From 88cd4762c339caf61ccb3fde25603e13aed56131 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Thu, 29 Jun 2023 16:18:49 +0100 Subject: [PATCH 001/377] fix: fixes the path for readiness probe --- config/internal/mcad/deployment.yaml.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/internal/mcad/deployment.yaml.tmpl b/config/internal/mcad/deployment.yaml.tmpl index 56cc691f..4620c1dc 100644 --- a/config/internal/mcad/deployment.yaml.tmpl +++ b/config/internal/mcad/deployment.yaml.tmpl @@ -48,7 +48,7 @@ spec: periodSeconds: 5 readinessProbe: httpGet: - path: /readyz + path: /healthz port: 8081 timeoutSeconds: 5 periodSeconds: 5 From f7a32268d9947aabb567db07fc5623c8477d6169 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 13 Jun 2023 12:01:42 +0200 Subject: [PATCH 002/377] Initial e2e tests --- .github/workflows/e2e_tests.yaml | 180 +++++++++++++++++++++++++++ .pre-commit-config.yaml | 1 - Makefile | 17 ++- go.mod | 2 + go.sum | 4 + test/e2e/kind.yaml | 32 +++++ test/e2e/mcad_test.go | 203 +++++++++++++++++++++++++++++++ test/support/client.go | 87 +++++++++++++ test/support/codeflare.go | 22 ++++ test/support/core.go | 17 +++ test/support/gomega.go | 28 +++++ test/support/mcad.go | 22 ++++ test/support/namespace.go | 56 +++++++++ test/support/support.go | 17 +++ test/support/test.go | 88 ++++++++++++++ test/support/utils.go | 5 + 16 files changed, 777 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/e2e_tests.yaml create mode 100644 test/e2e/kind.yaml create mode 100644 test/e2e/mcad_test.go create mode 100644 test/support/client.go create mode 100644 test/support/codeflare.go create mode 100644 test/support/core.go create mode 100644 test/support/gomega.go create mode 100644 test/support/mcad.go create mode 100644 test/support/namespace.go create mode 100644 test/support/support.go create mode 100644 test/support/test.go create mode 100644 test/support/utils.go diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml new file mode 100644 index 00000000..43e1c2ef --- /dev/null +++ b/.github/workflows/e2e_tests.yaml @@ -0,0 +1,180 @@ +name: e2e + +on: + pull_request: + branches: + - main + - 'release-*' + paths-ignore: + - 'docs/**' + - '**.adoc' + - '**.md' + - 'LICENSE' + push: + branches: + - main + - 'release-*' + paths-ignore: + - 'docs/**' + - '**.adoc' + - '**.md' + - 'LICENSE' + +concurrency: + group: ${{ github.head_ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + kubernetes: + + runs-on: ubuntu-20.04 + + steps: + - name: Cleanup + run: | + ls -lart + echo "Initial status:" + df -h + + echo "Cleaning up resources:" + sudo swapoff -a + sudo rm -f /swapfile + sudo apt clean + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + docker rmi $(docker image ls -aq) + + echo "Final status:" + df -h + + - name: Checkout code + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Init directories + run: | + TEMP_DIR="$(pwd)/tmp" + mkdir -p "${TEMP_DIR}" + echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV + + mkdir -p "$(pwd)/bin" + echo "$(pwd)/bin" >> $GITHUB_PATH + + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.18 + + - name: Container image registry + run: | + podman run -d -p 5000:5000 --name registry registry:2.8.1 + + export REGISTRY_ADDRESS=$(hostname -i):5000 + echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV + echo "Container image registry started at ${REGISTRY_ADDRESS}" + + KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml + echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV + envsubst < ./test/e2e/kind.yaml > ${KIND_CONFIG_FILE} + + sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf < Date: Mon, 19 Jun 2023 17:05:36 +0200 Subject: [PATCH 003/377] test: Submit MNIST RayJob --- go.mod | 1 + go.sum | 2 + test/e2e/mcad_test.go | 80 +++++++++++++++++++++- test/e2e/mnist.py | 148 +++++++++++++++++++++++++++++++++++++++++ test/e2e/support.go | 6 ++ test/support/client.go | 13 ++++ test/support/ray.go | 24 +++++++ 7 files changed, 271 insertions(+), 3 deletions(-) create mode 100644 test/e2e/mnist.py create mode 100644 test/e2e/support.go create mode 100644 test/support/ray.go diff --git a/go.mod b/go.mod index 621853fb..fff6657e 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/onsi/gomega v1.27.6 github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0 github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 + github.com/rs/xid v1.5.0 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index 8e877d1f..c475743c 100644 --- a/go.sum +++ b/go.sum @@ -465,6 +465,8 @@ github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 h github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9/go.mod h1:2auArgwD9dXXJz1oc7SqQ4U/rHdpwnrBwG98kr8OWXA= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= diff --git a/test/e2e/mcad_test.go b/test/e2e/mcad_test.go index 5db8bf7c..3c86f51a 100644 --- a/test/e2e/mcad_test.go +++ b/test/e2e/mcad_test.go @@ -18,6 +18,7 @@ limitations under the License. package e2e import ( + "encoding/base64" "testing" . "github.com/onsi/gomega" @@ -26,10 +27,11 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + "github.com/rs/xid" . "github.com/project-codeflare/codeflare-operator/test/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" ) func TestJobSubmissionInRayCluster(t *testing.T) { @@ -39,7 +41,29 @@ func TestJobSubmissionInRayCluster(t *testing.T) { // Create a namespace namespace := test.NewTestNamespace() + // Job script + mnist, err := scripts.ReadFile("mnist.py") + test.Expect(err).NotTo(HaveOccurred()) + + configMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.GroupName, + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + "mnist.py": mnist, + }, + Immutable: Ptr(true), + } + configMap, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), configMap, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + // RayCluster + clusterID := xid.New() rayCluster := &rayv1alpha1.RayCluster{ TypeMeta: metav1.TypeMeta{ APIVersion: rayv1alpha1.GroupVersion.String(), @@ -48,6 +72,9 @@ func TestJobSubmissionInRayCluster(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "raycluster-autoscaler", Namespace: namespace.Name, + Labels: map[string]string{ + RayJobDefaultClusterSelectorKey: clusterID.String(), + }, }, Spec: rayv1alpha1.RayClusterSpec{ RayVersion: "2.0.0", @@ -110,6 +137,24 @@ func TestJobSubmissionInRayCluster(t *testing.T) { corev1.ResourceMemory: resource.MustParse("1G"), }, }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "mnist", + MountPath: "/home/ray/jobs", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "mnist", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMap.Name, + }, + }, + }, }, }, }, @@ -195,9 +240,38 @@ func TestJobSubmissionInRayCluster(t *testing.T) { }, } - _, err := test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) + _, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) test.Expect(err).NotTo(HaveOccurred()) test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + rayJob := &rayv1alpha1.RayJob{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rayv1alpha1.GroupVersion.String(), + Kind: "RayJob", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + Spec: rayv1alpha1.RayJobSpec{ + Entrypoint: "python /home/ray/jobs/mnist.py", + RuntimeEnv: base64.StdEncoding.EncodeToString([]byte(` +pytorch_lightning==1.5.10 +ray_lightning +torchmetrics==0.9.1 +torchvision==0.12.0 +`)), + ClusterSelector: map[string]string{ + RayJobDefaultClusterSelectorKey: clusterID.String(), + }, + ShutdownAfterJobFinishes: false, + }, + } + _, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + test.Eventually(RayJob(test, namespace, rayJob.Name), TestTimeoutLong). + Should(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) } diff --git a/test/e2e/mnist.py b/test/e2e/mnist.py new file mode 100644 index 00000000..e60ec7c6 --- /dev/null +++ b/test/e2e/mnist.py @@ -0,0 +1,148 @@ +# In[] +import os + +import torch +from pytorch_lightning import LightningModule, Trainer +from pytorch_lightning.callbacks.progress import TQDMProgressBar +from pytorch_lightning.loggers import CSVLogger +from torch import nn +from torch.nn import functional as F +from torch.utils.data import DataLoader, random_split +from torchmetrics import Accuracy +from torchvision import transforms +from torchvision.datasets import MNIST + +PATH_DATASETS = os.environ.get("PATH_DATASETS", ".") +BATCH_SIZE = 256 if torch.cuda.is_available() else 64 +# %% + +print("prior to running the trainer") +print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) +print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) + + +class LitMNIST(LightningModule): + def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4): + + super().__init__() + + # Set our init args as class attributes + self.data_dir = data_dir + self.hidden_size = hidden_size + self.learning_rate = learning_rate + + # Hardcode some dataset specific attributes + self.num_classes = 10 + self.dims = (1, 28, 28) + channels, width, height = self.dims + self.transform = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)), + ] + ) + + # Define PyTorch model + self.model = nn.Sequential( + nn.Flatten(), + nn.Linear(channels * width * height, hidden_size), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden_size, hidden_size), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden_size, self.num_classes), + ) + + self.val_accuracy = Accuracy() + self.test_accuracy = Accuracy() + + def forward(self, x): + x = self.model(x) + return F.log_softmax(x, dim=1) + + def training_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + return loss + + def validation_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + preds = torch.argmax(logits, dim=1) + self.val_accuracy.update(preds, y) + + # Calling self.log will surface up scalars for you in TensorBoard + self.log("val_loss", loss, prog_bar=True) + self.log("val_acc", self.val_accuracy, prog_bar=True) + + def test_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + preds = torch.argmax(logits, dim=1) + self.test_accuracy.update(preds, y) + + # Calling self.log will surface up scalars for you in TensorBoard + self.log("test_loss", loss, prog_bar=True) + self.log("test_acc", self.test_accuracy, prog_bar=True) + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) + return optimizer + + #################### + # DATA RELATED HOOKS + #################### + + def prepare_data(self): + # download + print("Downloading MNIST dataset...") + MNIST(self.data_dir, train=True, download=True) + MNIST(self.data_dir, train=False, download=True) + + def setup(self, stage=None): + + # Assign train/val datasets for use in dataloaders + if stage == "fit" or stage is None: + mnist_full = MNIST(self.data_dir, train=True, transform=self.transform) + self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000]) + + # Assign test dataset for use in dataloader(s) + if stage == "test" or stage is None: + self.mnist_test = MNIST( + self.data_dir, train=False, transform=self.transform + ) + + def train_dataloader(self): + return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) + + def val_dataloader(self): + return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) + + def test_dataloader(self): + return DataLoader(self.mnist_test, batch_size=BATCH_SIZE) + + +# Init DataLoader from MNIST Dataset + +model = LitMNIST() + +print("GROUP: ", int(os.environ.get("GROUP_WORLD_SIZE", 1))) +print("LOCAL: ", int(os.environ.get("LOCAL_WORLD_SIZE", 1))) + +# Initialize a trainer +trainer = Trainer( + accelerator="auto", + # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs + max_epochs=5, + callbacks=[TQDMProgressBar(refresh_rate=20)], + num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), + devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), + strategy="ddp", +) + +# Train the model ⚡ +trainer.fit(model) diff --git a/test/e2e/support.go b/test/e2e/support.go new file mode 100644 index 00000000..27e9e54b --- /dev/null +++ b/test/e2e/support.go @@ -0,0 +1,6 @@ +package e2e + +import "embed" + +//go:embed *.py +var scripts embed.FS diff --git a/test/support/client.go b/test/support/client.go index 0026b293..634df4f1 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -27,18 +27,21 @@ import ( codeflareclient "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned" + rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" ) type Client interface { Core() kubernetes.Interface CodeFlare() codeflareclient.Interface MCAD() mcadclient.Interface + Ray() rayclient.Interface } type testClient struct { core kubernetes.Interface codeflare codeflareclient.Interface mcad mcadclient.Interface + ray rayclient.Interface } var _ Client = (*testClient)(nil) @@ -55,6 +58,10 @@ func (t *testClient) MCAD() mcadclient.Interface { return t.mcad } +func (t *testClient) Ray() rayclient.Interface { + return t.ray +} + func newTestClient() (Client, error) { cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( clientcmd.NewDefaultClientConfigLoadingRules(), @@ -79,9 +86,15 @@ func newTestClient() (Client, error) { return nil, err } + rayClient, err := rayclient.NewForConfig(cfg) + if err != nil { + return nil, err + } + return &testClient{ core: kubeClient, codeflare: codeFlareClient, mcad: mcadClient, + ray: rayClient, }, nil } diff --git a/test/support/ray.go b/test/support/ray.go new file mode 100644 index 00000000..dfc3b2bb --- /dev/null +++ b/test/support/ray.go @@ -0,0 +1,24 @@ +package support + +import ( + "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" +) + +const RayJobDefaultClusterSelectorKey = "ray.io/cluster" + +func RayJob(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayJob { + return func(g gomega.Gomega) *rayv1alpha1.RayJob { + job, err := t.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return job + } +} + +func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus { + return job.Status.JobStatus +} From aea8000c4bd6c084128a93d106d0055dad8e57cc Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 19 Jun 2023 17:25:31 +0200 Subject: [PATCH 004/377] test: Fix RayCluster labels selector --- go.mod | 1 - go.sum | 2 -- test/e2e/mcad_test.go | 8 +------- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/go.mod b/go.mod index fff6657e..621853fb 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/onsi/gomega v1.27.6 github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0 github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 - github.com/rs/xid v1.5.0 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index c475743c..8e877d1f 100644 --- a/go.sum +++ b/go.sum @@ -465,8 +465,6 @@ github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 h github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9/go.mod h1:2auArgwD9dXXJz1oc7SqQ4U/rHdpwnrBwG98kr8OWXA= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= diff --git a/test/e2e/mcad_test.go b/test/e2e/mcad_test.go index 3c86f51a..4557484d 100644 --- a/test/e2e/mcad_test.go +++ b/test/e2e/mcad_test.go @@ -27,8 +27,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/rs/xid" - . "github.com/project-codeflare/codeflare-operator/test/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" @@ -63,7 +61,6 @@ func TestJobSubmissionInRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) // RayCluster - clusterID := xid.New() rayCluster := &rayv1alpha1.RayCluster{ TypeMeta: metav1.TypeMeta{ APIVersion: rayv1alpha1.GroupVersion.String(), @@ -72,9 +69,6 @@ func TestJobSubmissionInRayCluster(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "raycluster-autoscaler", Namespace: namespace.Name, - Labels: map[string]string{ - RayJobDefaultClusterSelectorKey: clusterID.String(), - }, }, Spec: rayv1alpha1.RayClusterSpec{ RayVersion: "2.0.0", @@ -264,7 +258,7 @@ torchmetrics==0.9.1 torchvision==0.12.0 `)), ClusterSelector: map[string]string{ - RayJobDefaultClusterSelectorKey: clusterID.String(), + RayJobDefaultClusterSelectorKey: rayCluster.Name, }, ShutdownAfterJobFinishes: false, }, From 059af32eb3701fd1d875abd2dc3a6f3d2329f78e Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 19 Jun 2023 17:28:10 +0200 Subject: [PATCH 005/377] e2e: Print KubeRay operator logs --- .github/workflows/e2e_tests.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 43e1c2ef..1f1a80e1 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -178,3 +178,9 @@ jobs: run: | echo "Printing MCAD controller logs" kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher + + - name: Print KubeRay operator logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing KubeRay operator logs" + kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay From fb4b7b4ce98aa62830dfdb336c1b4e89f1227ccb Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 19 Jun 2023 17:44:58 +0200 Subject: [PATCH 006/377] test: Fix RayJob runtime environment --- test/e2e/mcad_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/e2e/mcad_test.go b/test/e2e/mcad_test.go index 4557484d..930222a6 100644 --- a/test/e2e/mcad_test.go +++ b/test/e2e/mcad_test.go @@ -252,10 +252,16 @@ func TestJobSubmissionInRayCluster(t *testing.T) { Spec: rayv1alpha1.RayJobSpec{ Entrypoint: "python /home/ray/jobs/mnist.py", RuntimeEnv: base64.StdEncoding.EncodeToString([]byte(` -pytorch_lightning==1.5.10 -ray_lightning -torchmetrics==0.9.1 -torchvision==0.12.0 +{ + "pip": [ + "pytorch_lightning==1.5.10", + "ray_lightning", + "torchmetrics==0.9.1", + "torchvision==0.12.0" + ], + "env_vars": { + } +} `)), ClusterSelector: map[string]string{ RayJobDefaultClusterSelectorKey: rayCluster.Name, From 922dd7a142e093fca493310a394701a88f8c022e Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 20 Jun 2023 10:33:00 +0200 Subject: [PATCH 007/377] test: Print RayJob logs --- test/e2e/mcad_test.go | 7 ++++++- test/support/ray.go | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/test/e2e/mcad_test.go b/test/e2e/mcad_test.go index 930222a6..263bbcb4 100644 --- a/test/e2e/mcad_test.go +++ b/test/e2e/mcad_test.go @@ -269,9 +269,14 @@ func TestJobSubmissionInRayCluster(t *testing.T) { ShutdownAfterJobFinishes: false, }, } - _, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) + rayJob, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.Eventually(RayJob(test, namespace, rayJob.Name), TestTimeoutLong). Should(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) + + rayJob, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Get(test.Ctx(), rayJob.Name, metav1.GetOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + test.T().Log(GetRayJobLogs(test, rayJob)) } diff --git a/test/support/ray.go b/test/support/ray.go index dfc3b2bb..3e1389ed 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -1,6 +1,8 @@ package support import ( + "encoding/json" + "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -22,3 +24,18 @@ func RayJob(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gome func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus { return job.Status.JobStatus } + +func GetRayJobLogs(t Test, job *rayv1alpha1.RayJob) string { + t.T().Helper() + response := t.Client().Core().CoreV1().RESTClient(). + Get(). + AbsPath("/api/v1/namespaces", job.Namespace, "services", "http:"+job.Status.RayClusterName+"-head-svc:dashboard", "proxy", "api", "jobs", job.Status.JobId, "logs").Do(t.Ctx()) + t.Expect(response.Error()).NotTo(gomega.HaveOccurred()) + + body := map[string]string{} + bytes, _ := response.Raw() + t.Expect(json.Unmarshal(bytes, &body)).To(gomega.Succeed()) + t.Expect(body).To(gomega.HaveKey("logs")) + + return body["logs"] +} From d4a6cfefdec035160d68ccf7b48be879ee7ffa75 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 21 Jun 2023 14:44:26 +0200 Subject: [PATCH 008/377] test: Document how to run e2e tests locally --- README.md | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index afc9dafa..31f21640 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # codeflare-operator + Operator for installation and lifecycle management of CodeFlare distributed workload stack, starting with MCAD and InstaScale @@ -14,7 +15,36 @@ CodeFlare Stack Compatibility Matrix | KubeRay | v0.5.0 | -## Release process +## Development + +### Testing + +The e2e tests can be executed locally by running the following commands: + +1. Setup the test cluster: + + ```bash + # Create a KinD cluster + $ kind create cluster --image kindest/node:v1.25.8 + # Install the CRDs + $ make install + ``` + +2. Start the operator locally: + + ```bash + $ make run + ``` + +3. In a separate terminal, run the e2e suite: + + ```bash + $ make test-e2e + ``` + + Alternatively, You can run the e2e test(s) from your IDE / debugger. + +## Release Prerequisite: - Build and release [MCAD](https://github.com/project-codeflare/multi-cluster-app-dispatcher) From ff609f9be997e6675a340a8334ebfaecb1e205f7 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 21 Jun 2023 15:24:11 +0200 Subject: [PATCH 009/377] test: Polish MNIST RayJob test --- test/e2e/kind.yaml | 23 +++++++++---------- ...o => mnist_rayjob_mcad_raycluster_test.go} | 15 ++++++------ test/e2e/support.go | 16 +++++++++++++ test/support/client.go | 15 ++++++------ test/support/codeflare.go | 22 ------------------ test/support/core.go | 16 +++++++++++++ test/support/gomega.go | 16 +++++++++++++ test/support/mcad.go | 16 +++++++++++++ test/support/namespace.go | 15 ++++++------ test/support/ray.go | 16 +++++++++++++ test/support/support.go | 16 +++++++++++++ test/support/test.go | 16 +++++++++++++ test/support/utils.go | 16 +++++++++++++ 13 files changed, 160 insertions(+), 58 deletions(-) rename test/e2e/{mcad_test.go => mnist_rayjob_mcad_raycluster_test.go} (94%) delete mode 100644 test/support/codeflare.go diff --git a/test/e2e/kind.yaml b/test/e2e/kind.yaml index ebb1f731..4546589b 100644 --- a/test/e2e/kind.yaml +++ b/test/e2e/kind.yaml @@ -1,18 +1,17 @@ # --------------------------------------------------------------------------- -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# Copyright 2023. # -# http://www.apache.org/licenses/LICENSE-2.0 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # --------------------------------------------------------------------------- kind: Cluster diff --git a/test/e2e/mcad_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go similarity index 94% rename from test/e2e/mcad_test.go rename to test/e2e/mnist_rayjob_mcad_raycluster_test.go index 263bbcb4..0fe82043 100644 --- a/test/e2e/mcad_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -1,12 +1,11 @@ /* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at +Copyright 2023. - http://www.apache.org/licenses/LICENSE-2.0 +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -32,7 +31,7 @@ import ( rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" ) -func TestJobSubmissionInRayCluster(t *testing.T) { +func TestMNISTRayJobMCADRayCluster(t *testing.T) { test := With(t) test.T().Parallel() diff --git a/test/e2e/support.go b/test/e2e/support.go index 27e9e54b..82f980eb 100644 --- a/test/e2e/support.go +++ b/test/e2e/support.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import "embed" diff --git a/test/support/client.go b/test/support/client.go index 634df4f1..a8efa76d 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -1,12 +1,11 @@ /* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/test/support/codeflare.go b/test/support/codeflare.go deleted file mode 100644 index 3c26caa2..00000000 --- a/test/support/codeflare.go +++ /dev/null @@ -1,22 +0,0 @@ -package support - -import ( - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -func MCAD(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *codeflarev1alpha1.MCAD { - return func(g gomega.Gomega) *codeflarev1alpha1.MCAD { - mcad, err := t.Client().CodeFlare().CodeflareV1alpha1().MCADs(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return mcad - } -} - -func ReadyStatus(mcad *codeflarev1alpha1.MCAD) bool { - return mcad.Status.Ready -} diff --git a/test/support/core.go b/test/support/core.go index 21c3537e..10fbd4b2 100644 --- a/test/support/core.go +++ b/test/support/core.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( diff --git a/test/support/gomega.go b/test/support/gomega.go index 27bb7223..5631044a 100644 --- a/test/support/gomega.go +++ b/test/support/gomega.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( diff --git a/test/support/mcad.go b/test/support/mcad.go index 0930fc38..4f268985 100644 --- a/test/support/mcad.go +++ b/test/support/mcad.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( diff --git a/test/support/namespace.go b/test/support/namespace.go index e9f57399..145acbb4 100644 --- a/test/support/namespace.go +++ b/test/support/namespace.go @@ -1,12 +1,11 @@ /* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/test/support/ray.go b/test/support/ray.go index 3e1389ed..369db80b 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( diff --git a/test/support/support.go b/test/support/support.go index 43a4951a..184b0186 100644 --- a/test/support/support.go +++ b/test/support/support.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( diff --git a/test/support/test.go b/test/support/test.go index 5fcb6f6f..5b3b271b 100644 --- a/test/support/test.go +++ b/test/support/test.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( diff --git a/test/support/utils.go b/test/support/utils.go index 9ddf287d..ed40309f 100644 --- a/test/support/utils.go +++ b/test/support/utils.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support func Ptr[T any](v T) *T { From 2361ef5216c36da1b1bc862c5705ac9c343748ed Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 09:50:47 +0200 Subject: [PATCH 010/377] test: Add MNIST training with MCAD Job --- test/e2e/mnist_pytorch_mcad_job_test.go | 182 ++++++++++++++++++ test/e2e/mnist_rayjob_mcad_raycluster_test.go | 2 +- test/support/batch.go | 33 ++++ test/support/conditions.go | 53 +++++ 4 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 test/e2e/mnist_pytorch_mcad_job_test.go create mode 100644 test/support/batch.go create mode 100644 test/support/conditions.go diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go new file mode 100644 index 00000000..242bef05 --- /dev/null +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -0,0 +1,182 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/project-codeflare/codeflare-operator/test/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" +) + +func TestMNISTPyTorchMCAD(t *testing.T) { + test := With(t) + test.T().Parallel() + + // Create a namespace + namespace := test.NewTestNamespace() + + // MNIST training script + mnist, err := scripts.ReadFile("mnist.py") + test.Expect(err).NotTo(HaveOccurred()) + + mnistScript := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + "mnist.py": mnist, + }, + Immutable: Ptr(true), + } + mnistScript, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnistScript, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + // pip requirements + requirements := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "requirements", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + "requirements.txt": []byte(` +pytorch_lightning==1.5.10 +torchmetrics==0.9.1 +torchvision==0.12.0 +`), + }, + Immutable: Ptr(true), + } + requirements, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), requirements, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + // Batch Job + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "job", + Image: "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime", + Command: []string{"/bin/sh", "-c", "pip install -r /test/runtime/requirements.txt && torchrun /test/job/mnist.py"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "mnist", + MountPath: "/test/job", + }, + { + Name: "requirements", + MountPath: "/test/runtime", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "mnist", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: mnistScript.Name, + }, + }, + }, + }, + { + Name: "requirements", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: requirements.Name, + }, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + } + + // Create an AppWrapper resource + aw := &mcadv1beta1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + Spec: mcadv1beta1.AppWrapperSpec{ + AggrResources: mcadv1beta1.AppWrapperResourceList{ + GenericItems: []mcadv1beta1.AppWrapperGenericResource{ + { + DesiredAvailable: 1, + CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ + { + Replicas: 1, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }, + }, + GenericTemplate: Raw(test, job), + }, + }, + }, + }, + } + + _, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) + test.Expect(err).NotTo(HaveOccurred()) + + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + test.Eventually(Job(test, namespace, job.Name), TestTimeoutLong). + Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) +} diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 0fe82043..0af97f5c 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -44,7 +44,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { configMap := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.GroupName, + APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ diff --git a/test/support/batch.go b/test/support/batch.go new file mode 100644 index 00000000..4aea06fd --- /dev/null +++ b/test/support/batch.go @@ -0,0 +1,33 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Job(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *batchv1.Job { + return func(g gomega.Gomega) *batchv1.Job { + job, err := t.Client().Core().BatchV1().Jobs(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return job + } +} diff --git a/test/support/conditions.go b/test/support/conditions.go new file mode 100644 index 00000000..e7c5097a --- /dev/null +++ b/test/support/conditions.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" +) + +type conditionType interface { + ~string +} + +func ConditionStatus[T conditionType](conditionType T) func(any) corev1.ConditionStatus { + return func(object any) corev1.ConditionStatus { + switch o := object.(type) { + + case *batchv1.Job: + if c := getJobCondition(o.Status.Conditions, batchv1.JobConditionType(conditionType)); c != nil { + return c.Status + } + + } + + return corev1.ConditionUnknown + } +} + +// TODO: to be replaced with a generic version once common struct fields of a type set can be used. +// See https://github.com/golang/go/issues/48522 +func getJobCondition(conditions []batchv1.JobCondition, conditionType batchv1.JobConditionType) *batchv1.JobCondition { + for _, c := range conditions { + if c.Type == conditionType { + return &c + } + } + return nil +} From 1285e8e8f5409bf70465de389e318ed53c48cbf8 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 10:54:53 +0200 Subject: [PATCH 011/377] test: Print MNIST batch job logs --- test/e2e/mnist_pytorch_mcad_job_test.go | 13 +++++++++++++ test/support/batch.go | 5 +++++ test/support/core.go | 26 +++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 242bef05..d305d9a6 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -25,6 +25,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" . "github.com/project-codeflare/codeflare-operator/test/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" @@ -179,4 +180,16 @@ torchvision==0.12.0 test.Eventually(Job(test, namespace, job.Name), TestTimeoutLong). Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + + // Refresh the job to get the generated pod selector + job = GetJob(test, namespace, job.Name) + + // Get the job Pod + pods := GetPods(test, namespace, metav1.ListOptions{ + LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, + ) + test.Expect(pods).To(HaveLen(1)) + + // Print the job logs + test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) } diff --git a/test/support/batch.go b/test/support/batch.go index 4aea06fd..c868d738 100644 --- a/test/support/batch.go +++ b/test/support/batch.go @@ -31,3 +31,8 @@ func Job(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) return job } } + +func GetJob(t Test, namespace *corev1.Namespace, name string) *batchv1.Job { + t.T().Helper() + return Job(t, namespace, name)(t) +} diff --git a/test/support/core.go b/test/support/core.go index 10fbd4b2..9da6d0b3 100644 --- a/test/support/core.go +++ b/test/support/core.go @@ -18,8 +18,12 @@ package support import ( "encoding/json" + "io" "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -31,3 +35,25 @@ func Raw(t Test, obj runtime.Object) runtime.RawExtension { Raw: data, } } + +func GetPods(t Test, namespace *corev1.Namespace, options metav1.ListOptions) []corev1.Pod { + t.T().Helper() + pods, err := t.Client().Core().CoreV1().Pods(namespace.Name).List(t.Ctx(), options) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return pods.Items +} + +func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) string { + t.T().Helper() + stream, err := t.Client().Core().CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetName(), &options).Stream(t.Ctx()) + t.Expect(err).NotTo(gomega.HaveOccurred()) + + defer func() { + t.Expect(stream.Close()).To(gomega.Succeed()) + }() + + bytes, err := io.ReadAll(stream) + t.Expect(err).NotTo(gomega.HaveOccurred()) + + return string(bytes) +} From 6553bfd2e7d591517241cadb405db655bd05e91f Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 11:07:21 +0200 Subject: [PATCH 012/377] test: Use RayCluster 'complete' configuration --- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 45 +++++-------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 0af97f5c..8e8f97f8 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -38,7 +38,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { // Create a namespace namespace := test.NewTestNamespace() - // Job script + // MNIST training script mnist, err := scripts.ReadFile("mnist.py") test.Expect(err).NotTo(HaveOccurred()) @@ -66,39 +66,21 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { Kind: "RayCluster", }, ObjectMeta: metav1.ObjectMeta{ - Name: "raycluster-autoscaler", + Name: "raycluster", Namespace: namespace.Name, }, Spec: rayv1alpha1.RayClusterSpec{ - RayVersion: "2.0.0", - EnableInTreeAutoscaling: Ptr(true), - AutoscalerOptions: &rayv1alpha1.AutoscalerOptions{ - UpscalingMode: Ptr[rayv1alpha1.UpscalingMode]("Default"), - IdleTimeoutSeconds: Ptr(int32(60)), - ImagePullPolicy: Ptr(corev1.PullAlways), - Resources: &corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - }, - }, + RayVersion: "2.0.0", HeadGroupSpec: rayv1alpha1.HeadGroupSpec{ RayStartParams: map[string]string{ "dashboard-host": "0.0.0.0", - "block": "true", }, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "ray-head", - Image: "rayproject/ray:2.0.0", - ImagePullPolicy: corev1.PullAlways, + Name: "ray-head", + Image: "rayproject/ray:2.0.0", Ports: []corev1.ContainerPort{ { ContainerPort: 6379, @@ -155,13 +137,11 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, WorkerGroupSpecs: []rayv1alpha1.WorkerGroupSpec{ { - Replicas: Ptr(int32(1)), - MinReplicas: Ptr(int32(1)), - MaxReplicas: Ptr(int32(3)), - GroupName: "small-group", - RayStartParams: map[string]string{ - "block": "true", - }, + Replicas: Ptr(int32(1)), + MinReplicas: Ptr(int32(1)), + MaxReplicas: Ptr(int32(2)), + GroupName: "small-group", + RayStartParams: map[string]string{}, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ InitContainers: []corev1.Container{ @@ -173,9 +153,8 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, Containers: []corev1.Container{ { - Name: "machine-learning", - Image: "rayproject/ray:2.0.0", - ImagePullPolicy: corev1.PullAlways, + Name: "ray-worker", + Image: "rayproject/ray:2.0.0", Lifecycle: &corev1.Lifecycle{ PreStop: &corev1.LifecycleHandler{ Exec: &corev1.ExecAction{ From 0c4e2aed7378c68583029b87b364b0fe6d3ebd33 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 11:17:25 +0200 Subject: [PATCH 013/377] test: Add step log statements --- test/e2e/mnist_pytorch_mcad_job_test.go | 6 ++++++ test/e2e/mnist_rayjob_mcad_raycluster_test.go | 14 ++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index d305d9a6..fcf5da43 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -58,6 +58,7 @@ func TestMNISTPyTorchMCAD(t *testing.T) { } mnistScript, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnistScript, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", mnistScript.Namespace, mnistScript.Name) // pip requirements requirements := &corev1.ConfigMap{ @@ -80,6 +81,7 @@ torchvision==0.12.0 } requirements, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), requirements, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", requirements.Namespace, requirements.Name) // Batch Job job := &batchv1.Job{ @@ -174,10 +176,13 @@ torchvision==0.12.0 _, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) + test.T().Logf("Waiting for MCAD %s/%s to be running", aw.Namespace, aw.Name) test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name) test.Eventually(Job(test, namespace, job.Name), TestTimeoutLong). Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) @@ -191,5 +196,6 @@ torchvision==0.12.0 test.Expect(pods).To(HaveLen(1)) // Print the job logs + test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name) test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) } diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 8e8f97f8..6bdba998 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -42,7 +42,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { mnist, err := scripts.ReadFile("mnist.py") test.Expect(err).NotTo(HaveOccurred()) - configMap := &corev1.ConfigMap{ + mnistScript := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", @@ -56,8 +56,9 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, Immutable: Ptr(true), } - configMap, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), configMap, metav1.CreateOptions{}) + mnistScript, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnistScript, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", mnistScript.Namespace, mnistScript.Name) // RayCluster rayCluster := &rayv1alpha1.RayCluster{ @@ -126,7 +127,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: configMap.Name, + Name: mnistScript.Name, }, }, }, @@ -212,9 +213,11 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, } - _, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) + aw, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) + test.T().Logf("Waiting for MCAD %s/%s to be running", aw.Namespace, aw.Name) test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) @@ -249,12 +252,15 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { } rayJob, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) + test.T().Logf("Waiting for RayJob %s/%s to complete successfully", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, namespace, rayJob.Name), TestTimeoutLong). Should(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) rayJob, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Get(test.Ctx(), rayJob.Name, metav1.GetOptions{}) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Printing RayJob %s/%s logs", rayJob.Namespace, rayJob.Name) test.T().Log(GetRayJobLogs(test, rayJob)) } From ae870ea3bbdadfa207cfd0c27776c5de872c235b Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 11:57:34 +0200 Subject: [PATCH 014/377] test: Add defered troubleshooting logs --- test/e2e/mnist_pytorch_mcad_job_test.go | 30 ++++++++++++++++++++++--- test/support/batch.go | 7 +++--- test/support/core.go | 4 ++-- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index fcf5da43..c4d6caf9 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -182,15 +182,17 @@ torchvision==0.12.0 test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + defer troubleshooting(test, job) + test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name) - test.Eventually(Job(test, namespace, job.Name), TestTimeoutLong). + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong). Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) // Refresh the job to get the generated pod selector - job = GetJob(test, namespace, job.Name) + job = GetJob(test, job.Namespace, job.Name) // Get the job Pod - pods := GetPods(test, namespace, metav1.ListOptions{ + pods := GetPods(test, job.Namespace, metav1.ListOptions{ LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, ) test.Expect(pods).To(HaveLen(1)) @@ -199,3 +201,25 @@ torchvision==0.12.0 test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name) test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) } + +func troubleshooting(test Test, job *batchv1.Job) { + if !test.T().Failed() { + return + } + job = GetJob(test, job.Namespace, job.Name) + + test.T().Errorf("Job %s/%s hasn't completed in time: %s", job.Namespace, job.Name, job) + + pods := GetPods(test, job.Namespace, metav1.ListOptions{ + LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, + ) + + if len(pods) == 0 { + test.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name) + } else { + for i, pod := range pods { + test.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name) + test.T().Log(GetPodLogs(test, &pods[i], corev1.PodLogOptions{})) + } + } +} diff --git a/test/support/batch.go b/test/support/batch.go index c868d738..2cb2d543 100644 --- a/test/support/batch.go +++ b/test/support/batch.go @@ -20,19 +20,18 @@ import ( "github.com/onsi/gomega" batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func Job(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *batchv1.Job { +func Job(t Test, namespace, name string) func(g gomega.Gomega) *batchv1.Job { return func(g gomega.Gomega) *batchv1.Job { - job, err := t.Client().Core().BatchV1().Jobs(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) + job, err := t.Client().Core().BatchV1().Jobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) return job } } -func GetJob(t Test, namespace *corev1.Namespace, name string) *batchv1.Job { +func GetJob(t Test, namespace, name string) *batchv1.Job { t.T().Helper() return Job(t, namespace, name)(t) } diff --git a/test/support/core.go b/test/support/core.go index 9da6d0b3..273c049a 100644 --- a/test/support/core.go +++ b/test/support/core.go @@ -36,9 +36,9 @@ func Raw(t Test, obj runtime.Object) runtime.RawExtension { } } -func GetPods(t Test, namespace *corev1.Namespace, options metav1.ListOptions) []corev1.Pod { +func GetPods(t Test, namespace string, options metav1.ListOptions) []corev1.Pod { t.T().Helper() - pods, err := t.Client().Core().CoreV1().Pods(namespace.Name).List(t.Ctx(), options) + pods, err := t.Client().Core().CoreV1().Pods(namespace).List(t.Ctx(), options) t.Expect(err).NotTo(gomega.HaveOccurred()) return pods.Items } From 4fb8a6ffba0934ab1b54689e12ef12b7253fd59d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 16:07:04 +0200 Subject: [PATCH 015/377] test: Add MNIST training in RayCluster with CodeFlare SDK --- test/e2e/mnist_pytorch_mcad_job_test.go | 37 +---- test/e2e/mnist_raycluster_sdk_test.go | 145 ++++++++++++++++++ test/e2e/mnist_rayjob_mcad_raycluster_test.go | 13 +- test/e2e/requirements.txt | 1 + test/e2e/sdk.py | 39 +++++ test/e2e/support.go | 18 ++- test/support/batch.go | 24 +++ 7 files changed, 235 insertions(+), 42 deletions(-) create mode 100644 test/e2e/mnist_raycluster_sdk_test.go create mode 100644 test/e2e/requirements.txt create mode 100644 test/e2e/sdk.py diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index c4d6caf9..a704d089 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -39,10 +39,7 @@ func TestMNISTPyTorchMCAD(t *testing.T) { namespace := test.NewTestNamespace() // MNIST training script - mnist, err := scripts.ReadFile("mnist.py") - test.Expect(err).NotTo(HaveOccurred()) - - mnistScript := &corev1.ConfigMap{ + mnist := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", @@ -52,13 +49,13 @@ func TestMNISTPyTorchMCAD(t *testing.T) { Namespace: namespace.Name, }, BinaryData: map[string][]byte{ - "mnist.py": mnist, + "mnist.py": ReadFile(test, "mnist.py"), }, Immutable: Ptr(true), } - mnistScript, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnistScript, metav1.CreateOptions{}) + mnist, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnist, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", mnistScript.Namespace, mnistScript.Name) + test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) // pip requirements requirements := &corev1.ConfigMap{ @@ -121,7 +118,7 @@ torchvision==0.12.0 VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: mnistScript.Name, + Name: mnist.Name, }, }, }, @@ -182,7 +179,7 @@ torchvision==0.12.0 test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - defer troubleshooting(test, job) + defer JobTroubleshooting(test, job) test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong). @@ -201,25 +198,3 @@ torchvision==0.12.0 test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name) test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) } - -func troubleshooting(test Test, job *batchv1.Job) { - if !test.T().Failed() { - return - } - job = GetJob(test, job.Namespace, job.Name) - - test.T().Errorf("Job %s/%s hasn't completed in time: %s", job.Namespace, job.Name, job) - - pods := GetPods(test, job.Namespace, metav1.ListOptions{ - LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, - ) - - if len(pods) == 0 { - test.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name) - } else { - for i, pod := range pods { - test.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name) - test.T().Log(GetPodLogs(test, &pods[i], corev1.PodLogOptions{})) - } - } -} diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go new file mode 100644 index 00000000..7283e463 --- /dev/null +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -0,0 +1,145 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/project-codeflare/codeflare-operator/test/support" +) + +func TestMNISTRayClusterSDK(t *testing.T) { + test := With(t) + test.T().Parallel() + + test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/146") + + // Create a namespace + namespace := test.NewTestNamespace() + + // SDK script + sdk := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + "sdk.py": ReadFile(test, "sdk.py"), + }, + Immutable: Ptr(true), + } + sdk, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), sdk, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", sdk.Namespace, sdk.Name) + + // pip requirements + requirements := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "requirements", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + "requirements.txt": ReadFile(test, "requirements.txt"), + }, + Immutable: Ptr(true), + } + requirements, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), requirements, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", requirements.Namespace, requirements.Name) + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + Namespace: namespace.Name, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + BackoffLimit: Ptr(int32(0)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "sdk", + Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Command: []string{"/bin/sh", "-c", "pip install -r /test/runtime/requirements.txt && python /test/job/sdk.py"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "sdk", + MountPath: "/test/job", + }, + { + Name: "requirements", + MountPath: "/test/runtime", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "sdk", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: sdk.Name, + }, + }, + }, + }, + { + Name: "requirements", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: requirements.Name, + }, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + } + job, err = test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + defer JobTroubleshooting(test, job) + + test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutMedium). + Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) +} diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 6bdba998..d7a6d18d 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -39,10 +39,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { namespace := test.NewTestNamespace() // MNIST training script - mnist, err := scripts.ReadFile("mnist.py") - test.Expect(err).NotTo(HaveOccurred()) - - mnistScript := &corev1.ConfigMap{ + mnist := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", @@ -52,13 +49,13 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { Namespace: namespace.Name, }, BinaryData: map[string][]byte{ - "mnist.py": mnist, + "mnist.py": ReadFile(test, "mnist.py"), }, Immutable: Ptr(true), } - mnistScript, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnistScript, metav1.CreateOptions{}) + mnist, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnist, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", mnistScript.Namespace, mnistScript.Name) + test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) // RayCluster rayCluster := &rayv1alpha1.RayCluster{ @@ -127,7 +124,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: mnistScript.Name, + Name: mnist.Name, }, }, }, diff --git a/test/e2e/requirements.txt b/test/e2e/requirements.txt new file mode 100644 index 00000000..382b9855 --- /dev/null +++ b/test/e2e/requirements.txt @@ -0,0 +1 @@ +codeflare-sdk==0.4.4 diff --git a/test/e2e/sdk.py b/test/e2e/sdk.py new file mode 100644 index 00000000..578bc48c --- /dev/null +++ b/test/e2e/sdk.py @@ -0,0 +1,39 @@ +from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration +# from codeflare_sdk.cluster.auth import TokenAuthentication +from codeflare_sdk.job.jobs import DDPJobDefinition + +cluster = Cluster(ClusterConfiguration( + name='mnist', + # namespace='default', + min_worker=1, + max_worker=1, + min_cpus=0.2, + max_cpus=1, + min_memory=0.5, + max_memory=1, + gpu=0, + instascale=False, +)) + +cluster.up() + +cluster.status() + +cluster.wait_ready() + +cluster.status() + +cluster.details() + +jobdef = DDPJobDefinition( + name="mnist", + script="/test/job/mnist.py", + scheduler_args={"requirements": "/test/runtime/requirements.txt"} +) +job = jobdef.submit(cluster) + +job.status() + +print(job.logs()) + +cluster.down() diff --git a/test/e2e/support.go b/test/e2e/support.go index 82f980eb..7847c075 100644 --- a/test/e2e/support.go +++ b/test/e2e/support.go @@ -16,7 +16,19 @@ limitations under the License. package e2e -import "embed" +import ( + "embed" -//go:embed *.py -var scripts embed.FS + "github.com/onsi/gomega" + + "github.com/project-codeflare/codeflare-operator/test/support" +) + +//go:embed *.py *.txt +var files embed.FS + +func ReadFile(t support.Test, fileName string) []byte { + file, err := files.ReadFile(fileName) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return file +} diff --git a/test/support/batch.go b/test/support/batch.go index 2cb2d543..1bf6874e 100644 --- a/test/support/batch.go +++ b/test/support/batch.go @@ -20,7 +20,9 @@ import ( "github.com/onsi/gomega" batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" ) func Job(t Test, namespace, name string) func(g gomega.Gomega) *batchv1.Job { @@ -35,3 +37,25 @@ func GetJob(t Test, namespace, name string) *batchv1.Job { t.T().Helper() return Job(t, namespace, name)(t) } + +func JobTroubleshooting(test Test, job *batchv1.Job) { + if !test.T().Failed() { + return + } + job = GetJob(test, job.Namespace, job.Name) + + test.T().Errorf("Job %s/%s hasn't completed in time: %s", job.Namespace, job.Name, job) + + pods := GetPods(test, job.Namespace, metav1.ListOptions{ + LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, + ) + + if len(pods) == 0 { + test.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name) + } else { + for i, pod := range pods { + test.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name) + test.T().Log(GetPodLogs(test, &pods[i], corev1.PodLogOptions{})) + } + } +} From 3c211432d2fb0c4c3e96995d26c9a7e9b5670919 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 22 Jun 2023 17:25:58 +0200 Subject: [PATCH 016/377] test: Customize test timeouts --- .github/workflows/e2e_tests.yaml | 4 ++++ test/support/gomega.go | 13 ---------- test/support/support.go | 41 +++++++++++++++++++++++++++++--- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 1f1a80e1..1f8263d5 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -165,6 +165,10 @@ jobs: - name: Run e2e tests run: | + export CODEFLARE_TEST_TIMEOUT_SHORT=1m + export CODEFLARE_TEST_TIMEOUT_MEDIUM=3m + export CODEFLARE_TEST_TIMEOUT_LONG=8m + make test-e2e - name: Print CodeFlare operator logs diff --git a/test/support/gomega.go b/test/support/gomega.go index 5631044a..74897502 100644 --- a/test/support/gomega.go +++ b/test/support/gomega.go @@ -17,24 +17,11 @@ limitations under the License. package support import ( - "time" - "github.com/onsi/gomega" - "github.com/onsi/gomega/format" "github.com/onsi/gomega/gstruct" "github.com/onsi/gomega/types" ) -func init() { - // Gomega settings - gomega.SetDefaultEventuallyTimeout(TestTimeoutShort) - gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) - gomega.SetDefaultConsistentlyDuration(30 * time.Second) - gomega.SetDefaultConsistentlyPollingInterval(1 * time.Second) - // Disable object truncation on test results - format.MaxLength = 0 -} - func EqualP(expected interface{}) types.GomegaMatcher { return gstruct.PointTo(gomega.Equal(expected)) } diff --git a/test/support/support.go b/test/support/support.go index 184b0186..782e65e7 100644 --- a/test/support/support.go +++ b/test/support/support.go @@ -17,17 +17,52 @@ limitations under the License. package support import ( + "fmt" + "os" "time" + "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -const ( +var ( TestTimeoutShort = 1 * time.Minute TestTimeoutMedium = 2 * time.Minute TestTimeoutLong = 5 * time.Minute -) -var ( ApplyOptions = metav1.ApplyOptions{FieldManager: "codeflare-test", Force: true} ) + +func init() { + if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_SHORT"); ok { + if duration, err := time.ParseDuration(value); err == nil { + TestTimeoutShort = duration + } else { + fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_SHORT. Using default value: %s", TestTimeoutShort) + } + } + if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_MEDIUM"); ok { + if duration, err := time.ParseDuration(value); err == nil { + TestTimeoutMedium = duration + } else { + fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_MEDIUM. Using default value: %s", TestTimeoutMedium) + } + } + if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_LONG"); ok { + if duration, err := time.ParseDuration(value); err == nil { + TestTimeoutLong = duration + } else { + fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_LONG. Using default value: %s", TestTimeoutLong) + } + } + + // Gomega settings + gomega.SetDefaultEventuallyTimeout(TestTimeoutShort) + gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) + gomega.SetDefaultConsistentlyDuration(30 * time.Second) + gomega.SetDefaultConsistentlyPollingInterval(1 * time.Second) + // Disable object truncation on test results + format.MaxLength = 0 +} From fc86a757714c6af7112abbfe88578e1cc9ca469d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 26 Jun 2023 17:45:11 +0200 Subject: [PATCH 017/377] test: Pass MNIST training with CodeFlare SDK on OpenShift --- test/e2e/mnist.py | 16 ++- test/e2e/mnist_raycluster_sdk.py | 65 +++++++++++ test/e2e/mnist_raycluster_sdk_test.go | 150 +++++++++++++++++++------- test/e2e/requirements.txt | 4 +- test/e2e/sdk.py | 39 ------- test/support/openshift.go | 33 ++++++ 6 files changed, 225 insertions(+), 82 deletions(-) create mode 100644 test/e2e/mnist_raycluster_sdk.py delete mode 100644 test/e2e/sdk.py create mode 100644 test/support/openshift.go diff --git a/test/e2e/mnist.py b/test/e2e/mnist.py index e60ec7c6..244c84d2 100644 --- a/test/e2e/mnist.py +++ b/test/e2e/mnist.py @@ -1,10 +1,22 @@ -# In[] +# Copyright 2022 IBM, Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import torch from pytorch_lightning import LightningModule, Trainer from pytorch_lightning.callbacks.progress import TQDMProgressBar -from pytorch_lightning.loggers import CSVLogger from torch import nn from torch.nn import functional as F from torch.utils.data import DataLoader, random_split diff --git a/test/e2e/mnist_raycluster_sdk.py b/test/e2e/mnist_raycluster_sdk.py new file mode 100644 index 00000000..444b1858 --- /dev/null +++ b/test/e2e/mnist_raycluster_sdk.py @@ -0,0 +1,65 @@ +import sys + +from time import sleep + +from torchx.specs.api import AppState, is_terminal + +from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration +from codeflare_sdk.job.jobs import DDPJobDefinition + +namespace = sys.argv[1] + +cluster = Cluster(ClusterConfiguration( + name='mnist', + namespace=namespace, + min_worker=1, + max_worker=1, + min_cpus='500m', + max_cpus=1, + min_memory=0.5, + max_memory=1, + gpu=0, + instascale=False, +)) + +cluster.up() + +cluster.status() + +cluster.wait_ready() + +cluster.status() + +cluster.details() + +jobdef = DDPJobDefinition( + name="mnist", + script="mnist.py", + scheduler_args={"requirements": "requirements.txt"}, +) +job = jobdef.submit(cluster) + +done = False +time = 0 +timeout = 300 +while not done: + status = job.status() + if is_terminal(status.state): + break + if not done: + print(status) + if timeout and time >= timeout: + raise TimeoutError(f"job has timed out after waiting {timeout}s") + sleep(5) + time += 5 + +print(f"Job has completed: {status.state}") + +print(job.logs()) + +cluster.down() + +if not status.state == AppState.SUCCEEDED: + exit(1) +else: + exit(0) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 7283e463..43854e79 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -23,57 +23,119 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" . "github.com/project-codeflare/codeflare-operator/test/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) func TestMNISTRayClusterSDK(t *testing.T) { test := With(t) test.T().Parallel() - test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/146") + if !IsOpenShift(test) { + test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/146") + } // Create a namespace namespace := test.NewTestNamespace() - // SDK script - sdk := &corev1.ConfigMap{ + // Test configuration + configMap := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", + Name: "mnist-raycluster-sdk", Namespace: namespace.Name, }, BinaryData: map[string][]byte{ - "sdk.py": ReadFile(test, "sdk.py"), + // SDK script + "mnist_raycluster_sdk.py": ReadFile(test, "mnist_raycluster_sdk.py"), + // pip requirements + "requirements.txt": ReadFile(test, "requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), }, Immutable: Ptr(true), } - sdk, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), sdk, metav1.CreateOptions{}) + configMap, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), configMap, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", sdk.Namespace, sdk.Name) + test.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) - // pip requirements - requirements := &corev1.ConfigMap{ + // SDK client RBAC + serviceAccount := &corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", + Kind: "ServiceAccount", }, ObjectMeta: metav1.ObjectMeta{ - Name: "requirements", + Name: "sdk-user", Namespace: namespace.Name, }, - BinaryData: map[string][]byte{ - "requirements.txt": ReadFile(test, "requirements.txt"), + } + serviceAccount, err = test.Client().Core().CoreV1().ServiceAccounts(namespace.Name).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + role := &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "Role", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + Namespace: namespace.Name, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1alpha1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, + }, }, - Immutable: Ptr(true), } - requirements, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), requirements, metav1.CreateOptions{}) + role, err = test.Client().Core().RbacV1().Roles(namespace.Name).Create(test.Ctx(), role, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + roleBinding := &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "Role", + Name: role.Name, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + APIGroup: corev1.SchemeGroupVersion.Group, + Name: serviceAccount.Name, + Namespace: serviceAccount.Namespace, + }, + }, + } + _, err = test.Client().Core().RbacV1().RoleBindings(namespace.Name).Create(test.Ctx(), roleBinding, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", requirements.Namespace, requirements.Name) job := &batchv1.Job{ TypeMeta: metav1.TypeMeta{ @@ -92,54 +154,62 @@ func TestMNISTRayClusterSDK(t *testing.T) { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "sdk", + Name: "test", Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", - Command: []string{"/bin/sh", "-c", "pip install -r /test/runtime/requirements.txt && python /test/job/sdk.py"}, + Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==0.4.4 && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, VolumeMounts: []corev1.VolumeMount{ { - Name: "sdk", - MountPath: "/test/job", - }, - { - Name: "requirements", - MountPath: "/test/runtime", + Name: "test", + MountPath: "/test", }, }, }, }, Volumes: []corev1.Volume{ { - Name: "sdk", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: sdk.Name, - }, - }, - }, - }, - { - Name: "requirements", + Name: "test", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: requirements.Name, + Name: configMap.Name, }, }, }, }, }, - RestartPolicy: corev1.RestartPolicyNever, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: serviceAccount.Name, }, }, }, } job, err = test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) defer JobTroubleshooting(test, job) - test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutMedium). - Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + // Refresh the job to get the generated pod selector + job = GetJob(test, job.Namespace, job.Name) + + // Get the job Pod + pods := GetPods(test, job.Namespace, metav1.ListOptions{ + LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, + ) + test.Expect(pods).To(HaveLen(1)) + + // Print the job logs + test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name) + test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) + + // Assert the job has completed successfully + test.T().Logf("Checking the Job %s/%s has completed successfully", job.Namespace, job.Name) + test.Expect(job).To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) } diff --git a/test/e2e/requirements.txt b/test/e2e/requirements.txt index 382b9855..87edeef2 100644 --- a/test/e2e/requirements.txt +++ b/test/e2e/requirements.txt @@ -1 +1,3 @@ -codeflare-sdk==0.4.4 +pytorch_lightning==1.5.10 +torchmetrics==0.9.1 +torchvision==0.12.0 diff --git a/test/e2e/sdk.py b/test/e2e/sdk.py deleted file mode 100644 index 578bc48c..00000000 --- a/test/e2e/sdk.py +++ /dev/null @@ -1,39 +0,0 @@ -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration -# from codeflare_sdk.cluster.auth import TokenAuthentication -from codeflare_sdk.job.jobs import DDPJobDefinition - -cluster = Cluster(ClusterConfiguration( - name='mnist', - # namespace='default', - min_worker=1, - max_worker=1, - min_cpus=0.2, - max_cpus=1, - min_memory=0.5, - max_memory=1, - gpu=0, - instascale=False, -)) - -cluster.up() - -cluster.status() - -cluster.wait_ready() - -cluster.status() - -cluster.details() - -jobdef = DDPJobDefinition( - name="mnist", - script="/test/job/mnist.py", - scheduler_args={"requirements": "/test/runtime/requirements.txt"} -) -job = jobdef.submit(cluster) - -job.status() - -print(job.logs()) - -cluster.down() diff --git a/test/support/openshift.go b/test/support/openshift.go new file mode 100644 index 00000000..cfe3b5a3 --- /dev/null +++ b/test/support/openshift.go @@ -0,0 +1,33 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/errors" +) + +func IsOpenShift(test Test) bool { + test.T().Helper() + _, err := test.Client().Core().Discovery().ServerResourcesForGroupVersion("image.openshift.io/v1") + if err != nil && errors.IsNotFound(err) { + return false + } + test.Expect(err).NotTo(gomega.HaveOccurred()) + return true +} From f55e8c98c6250f8c1e34f89c3c8f12a93cae512e Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 10:43:58 +0200 Subject: [PATCH 018/377] test: Print Job logs after successfull or failed completion --- test/e2e/mnist_pytorch_mcad_job_test.go | 28 ++++++++++--------------- test/e2e/mnist_raycluster_sdk_test.go | 19 +++-------------- test/support/batch.go | 17 +++++++-------- 3 files changed, 21 insertions(+), 43 deletions(-) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index a704d089..778fe106 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -25,7 +25,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" . "github.com/project-codeflare/codeflare-operator/test/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" @@ -179,22 +178,17 @@ torchvision==0.12.0 test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - defer JobTroubleshooting(test, job) - - test.T().Logf("Waiting for Job %s/%s to complete successfully", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong). - Should(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) - - // Refresh the job to get the generated pod selector - job = GetJob(test, job.Namespace, job.Name) - - // Get the job Pod - pods := GetPods(test, job.Namespace, metav1.ListOptions{ - LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, - ) - test.Expect(pods).To(HaveLen(1)) + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) // Print the job logs - test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name) - test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) + PrintJobLogs(test, job.Namespace, job.Name) + + // Assert the job has completed successfully + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) } diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 43854e79..3761c935 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -25,7 +25,6 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" @@ -187,8 +186,6 @@ func TestMNISTRayClusterSDK(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) - defer JobTroubleshooting(test, job) - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( Or( @@ -196,20 +193,10 @@ func TestMNISTRayClusterSDK(t *testing.T) { WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), )) - // Refresh the job to get the generated pod selector - job = GetJob(test, job.Namespace, job.Name) - - // Get the job Pod - pods := GetPods(test, job.Namespace, metav1.ListOptions{ - LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, - ) - test.Expect(pods).To(HaveLen(1)) - // Print the job logs - test.T().Logf("Printing Job %s/%s logs", job.Namespace, job.Name) - test.T().Log(GetPodLogs(test, &pods[0], corev1.PodLogOptions{})) + PrintJobLogs(test, job.Namespace, job.Name) // Assert the job has completed successfully - test.T().Logf("Checking the Job %s/%s has completed successfully", job.Namespace, job.Name) - test.Expect(job).To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) } diff --git a/test/support/batch.go b/test/support/batch.go index 1bf6874e..6349efa5 100644 --- a/test/support/batch.go +++ b/test/support/batch.go @@ -38,24 +38,21 @@ func GetJob(t Test, namespace, name string) *batchv1.Job { return Job(t, namespace, name)(t) } -func JobTroubleshooting(test Test, job *batchv1.Job) { - if !test.T().Failed() { - return - } - job = GetJob(test, job.Namespace, job.Name) +func PrintJobLogs(t Test, namespace, name string) { + t.T().Helper() - test.T().Errorf("Job %s/%s hasn't completed in time: %s", job.Namespace, job.Name, job) + job := GetJob(t, namespace, name) - pods := GetPods(test, job.Namespace, metav1.ListOptions{ + pods := GetPods(t, job.Namespace, metav1.ListOptions{ LabelSelector: labels.FormatLabels(job.Spec.Selector.MatchLabels)}, ) if len(pods) == 0 { - test.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name) + t.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name) } else { for i, pod := range pods { - test.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name) - test.T().Log(GetPodLogs(test, &pods[i], corev1.PodLogOptions{})) + t.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name) + t.T().Log(GetPodLogs(t, &pods[i], corev1.PodLogOptions{})) } } } From 24841ca3936c4c8c82325065af332ec76992a698 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 10:54:09 +0200 Subject: [PATCH 019/377] test: Re-use pip requirements file --- ...rements.txt => mnist_pip_requirements.txt} | 0 test/e2e/mnist_pytorch_mcad_job_test.go | 60 ++++--------------- test/e2e/mnist_raycluster_sdk_test.go | 10 ++-- 3 files changed, 18 insertions(+), 52 deletions(-) rename test/e2e/{requirements.txt => mnist_pip_requirements.txt} (100%) diff --git a/test/e2e/requirements.txt b/test/e2e/mnist_pip_requirements.txt similarity index 100% rename from test/e2e/requirements.txt rename to test/e2e/mnist_pip_requirements.txt diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 778fe106..f1664a53 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -37,47 +37,27 @@ func TestMNISTPyTorchMCAD(t *testing.T) { // Create a namespace namespace := test.NewTestNamespace() - // MNIST training script - mnist := &corev1.ConfigMap{ + // Test configuration + config := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ - Name: "mnist", + Name: "mnist-mcad", Namespace: namespace.Name, }, BinaryData: map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script "mnist.py": ReadFile(test, "mnist.py"), }, Immutable: Ptr(true), } - mnist, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), mnist, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) - - // pip requirements - requirements := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "requirements", - Namespace: namespace.Name, - }, - BinaryData: map[string][]byte{ - "requirements.txt": []byte(` -pytorch_lightning==1.5.10 -torchmetrics==0.9.1 -torchvision==0.12.0 -`), - }, - Immutable: Ptr(true), - } - requirements, err = test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), requirements, metav1.CreateOptions{}) + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", requirements.Namespace, requirements.Name) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) // Batch Job job := &batchv1.Job{ @@ -98,36 +78,22 @@ torchvision==0.12.0 { Name: "job", Image: "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime", - Command: []string{"/bin/sh", "-c", "pip install -r /test/runtime/requirements.txt && torchrun /test/job/mnist.py"}, + Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ { - Name: "mnist", - MountPath: "/test/job", - }, - { - Name: "requirements", - MountPath: "/test/runtime", + Name: "test", + MountPath: "/test", }, }, }, }, Volumes: []corev1.Volume{ { - Name: "mnist", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: mnist.Name, - }, - }, - }, - }, - { - Name: "requirements", + Name: "test", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: requirements.Name, + Name: config.Name, }, }, }, diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 3761c935..82087155 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -44,7 +44,7 @@ func TestMNISTRayClusterSDK(t *testing.T) { namespace := test.NewTestNamespace() // Test configuration - configMap := &corev1.ConfigMap{ + config := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", @@ -57,15 +57,15 @@ func TestMNISTRayClusterSDK(t *testing.T) { // SDK script "mnist_raycluster_sdk.py": ReadFile(test, "mnist_raycluster_sdk.py"), // pip requirements - "requirements.txt": ReadFile(test, "requirements.txt"), + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), // MNIST training script "mnist.py": ReadFile(test, "mnist.py"), }, Immutable: Ptr(true), } - configMap, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), configMap, metav1.CreateOptions{}) + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) // SDK client RBAC serviceAccount := &corev1.ServiceAccount{ @@ -170,7 +170,7 @@ func TestMNISTRayClusterSDK(t *testing.T) { VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: configMap.Name, + Name: config.Name, }, }, }, From 475137e4e2b37eb8b786395aa81a09e15d2343f8 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 15:20:26 +0200 Subject: [PATCH 020/377] test: Parameterize CodeFlare SDK version --- Makefile | 4 ++++ controllers/defaults.go | 5 +++-- test/e2e/mnist_raycluster_sdk_test.go | 2 +- test/support/codeflare.go | 30 +++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 test/support/codeflare.go diff --git a/Makefile b/Makefile index c9295f57..2185264e 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,9 @@ MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_REF} # KUBERAY_VERSION defines the default version of the KubeRay operator KUBERAY_VERSION ?= v0.5.0 +# CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK +CODEFLARE_SDK_VERSION ?= 0.4.4 + # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to OPERATORS_REPO_ORG ?= redhat-openshift-ecosystem @@ -132,6 +135,7 @@ defaults: @echo "const (" >> $(DEFAULTS_FILE) @echo " MCADImage = \"$(MCAD_IMAGE)\"" >> $(DEFAULTS_FILE) @echo " InstaScaleImage = \"$(INSTASCALE_IMAGE)\"" >> $(DEFAULTS_FILE) + @echo " CodeFlareSDKVersion = \"$(CODEFLARE_SDK_VERSION)\"" >> $(DEFAULTS_FILE) @echo "" >> $(DEFAULTS_FILE) @echo ")" >> $(DEFAULTS_FILE) @echo "" >> $(DEFAULTS_FILE) diff --git a/controllers/defaults.go b/controllers/defaults.go index d3cac3d0..497940a9 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,6 +5,7 @@ package controllers // *********************** const ( - MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.31.0" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.4" + MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.31.0" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.4" + CodeFlareSDKVersion = "0.4.4" ) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 82087155..3ba168a0 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -155,7 +155,7 @@ func TestMNISTRayClusterSDK(t *testing.T) { { Name: "test", Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", - Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==0.4.4 && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, + Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, VolumeMounts: []corev1.VolumeMount{ { Name: "test", diff --git a/test/support/codeflare.go b/test/support/codeflare.go new file mode 100644 index 00000000..69066c2a --- /dev/null +++ b/test/support/codeflare.go @@ -0,0 +1,30 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "os" + + "github.com/project-codeflare/codeflare-operator/controllers" +) + +func GetCodeFlareSDKVersion() string { + if value, ok := os.LookupEnv("CODEFLARE_SDK_VERSION"); ok { + return value + } + return controllers.CodeFlareSDKVersion +} From f0960c1530d4d3782c8b4238c9a0915019960e23 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 15:58:06 +0200 Subject: [PATCH 021/377] test: Remove ray_lightning from requirements --- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 1 - test/support/ray.go | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index d7a6d18d..38a3b97c 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -233,7 +233,6 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { { "pip": [ "pytorch_lightning==1.5.10", - "ray_lightning", "torchmetrics==0.9.1", "torchvision==0.12.0" ], diff --git a/test/support/ray.go b/test/support/ray.go index 369db80b..9e0d22d9 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -45,7 +45,8 @@ func GetRayJobLogs(t Test, job *rayv1alpha1.RayJob) string { t.T().Helper() response := t.Client().Core().CoreV1().RESTClient(). Get(). - AbsPath("/api/v1/namespaces", job.Namespace, "services", "http:"+job.Status.RayClusterName+"-head-svc:dashboard", "proxy", "api", "jobs", job.Status.JobId, "logs").Do(t.Ctx()) + AbsPath("/api/v1/namespaces", job.Namespace, "services", "http:"+job.Status.RayClusterName+"-head-svc:dashboard", "proxy", "api", "jobs", job.Status.JobId, "logs"). + Do(t.Ctx()) t.Expect(response.Error()).NotTo(gomega.HaveOccurred()) body := map[string]string{} From 3fd6af66dc84ad5c2033bf34579b5693897032f5 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 16:35:59 +0200 Subject: [PATCH 022/377] test: Parameterize Ray image and version --- Makefile | 27 ++++++++++++++++--- controllers/defaults.go | 5 ++-- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 6 ++--- test/support/codeflare.go | 26 +++++++++++++++--- test/support/defaults.go | 11 ++++++++ 5 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 test/support/defaults.go diff --git a/Makefile b/Makefile index 2185264e..c3ce8665 100644 --- a/Makefile +++ b/Makefile @@ -21,9 +21,12 @@ MCAD_REF ?= release-${MCAD_VERSION} MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_REF} -# KUBERAY_VERSION defines the default version of the KubeRay operator +# KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) KUBERAY_VERSION ?= v0.5.0 +# RAY_VERSION defines the default version of Ray (used for testing) +RAY_VERSION ?= 2.5.0 + # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK CODEFLARE_SDK_VERSION ?= 0.4.4 @@ -67,6 +70,9 @@ MCAD_IMAGE ?= $(IMAGE_ORG_BASE)/mcad-controller:$(MCAD_REF) # INSTASCALE_IMAGE defines the default container image for the InstaScale controller INSTASCALE_IMAGE ?= $(IMAGE_ORG_BASE)/instascale-controller:$(INSTASCALE_VERSION) +# RAY_IMAGE defines the default container image for Ray (used for testing) +RAY_IMAGE ?= rayproject/ray:$(RAY_VERSION) + # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(VERSION) @@ -122,6 +128,7 @@ help: ## Display this help. ##@ Development DEFAULTS_FILE := controllers/defaults.go +DEFAULTS_TEST_FILE := test/support/defaults.go .PHONY: defaults defaults: @@ -135,12 +142,26 @@ defaults: @echo "const (" >> $(DEFAULTS_FILE) @echo " MCADImage = \"$(MCAD_IMAGE)\"" >> $(DEFAULTS_FILE) @echo " InstaScaleImage = \"$(INSTASCALE_IMAGE)\"" >> $(DEFAULTS_FILE) - @echo " CodeFlareSDKVersion = \"$(CODEFLARE_SDK_VERSION)\"" >> $(DEFAULTS_FILE) @echo "" >> $(DEFAULTS_FILE) @echo ")" >> $(DEFAULTS_FILE) @echo "" >> $(DEFAULTS_FILE) - gofmt -w $(DEFAULTS_FILE) + $(info Regenerating $(DEFAULTS_TEST_FILE)) + @echo "package support" > $(DEFAULTS_TEST_FILE) + @echo "" >> $(DEFAULTS_TEST_FILE) + @echo "// ***********************" >> $(DEFAULTS_TEST_FILE) + @echo "// DO NOT EDIT THIS FILE" >> $(DEFAULTS_TEST_FILE) + @echo "// ***********************" >> $(DEFAULTS_TEST_FILE) + @echo "" >> $(DEFAULTS_TEST_FILE) + @echo "const (" >> $(DEFAULTS_TEST_FILE) + @echo " CodeFlareSDKVersion = \"$(CODEFLARE_SDK_VERSION)\"" >> $(DEFAULTS_TEST_FILE) + @echo " RayVersion = \"$(RAY_VERSION)\"" >> $(DEFAULTS_TEST_FILE) + @echo " RayImage = \"$(RAY_IMAGE)\"" >> $(DEFAULTS_TEST_FILE) + @echo "" >> $(DEFAULTS_TEST_FILE) + @echo ")" >> $(DEFAULTS_TEST_FILE) + @echo "" >> $(DEFAULTS_TEST_FILE) + + gofmt -w $(DEFAULTS_FILE) $(DEFAULTS_TEST_FILE) .PHONY: manifests manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. diff --git a/controllers/defaults.go b/controllers/defaults.go index 497940a9..d3cac3d0 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,7 +5,6 @@ package controllers // *********************** const ( - MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.31.0" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.4" - CodeFlareSDKVersion = "0.4.4" + MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.31.0" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.4" ) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 38a3b97c..33a9e6a5 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -68,7 +68,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { Namespace: namespace.Name, }, Spec: rayv1alpha1.RayClusterSpec{ - RayVersion: "2.0.0", + RayVersion: GetRayVersion(), HeadGroupSpec: rayv1alpha1.HeadGroupSpec{ RayStartParams: map[string]string{ "dashboard-host": "0.0.0.0", @@ -78,7 +78,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { Containers: []corev1.Container{ { Name: "ray-head", - Image: "rayproject/ray:2.0.0", + Image: GetRayImage(), Ports: []corev1.ContainerPort{ { ContainerPort: 6379, @@ -152,7 +152,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { Containers: []corev1.Container{ { Name: "ray-worker", - Image: "rayproject/ray:2.0.0", + Image: GetRayImage(), Lifecycle: &corev1.Lifecycle{ PreStop: &corev1.LifecycleHandler{ Exec: &corev1.ExecAction{ diff --git a/test/support/codeflare.go b/test/support/codeflare.go index 69066c2a..0763bbd3 100644 --- a/test/support/codeflare.go +++ b/test/support/codeflare.go @@ -18,13 +18,31 @@ package support import ( "os" +) - "github.com/project-codeflare/codeflare-operator/controllers" +// The environment variables hereafter can be used to change the components +// used for testing. +const ( + CodeFlareTestSdkVersion = "CODEFLARE_TEST_SDK_VERSION" + CodeFlareTestRayVersion = "CODEFLARE_TEST_RAY_VERSION" + CodeFlareTestRayImage = "CODEFLARE_TEST_RAY_IMAGE" ) func GetCodeFlareSDKVersion() string { - if value, ok := os.LookupEnv("CODEFLARE_SDK_VERSION"); ok { - return value + return lookupEnvOrDefault(CodeFlareTestSdkVersion, CodeFlareSDKVersion) +} + +func GetRayVersion() string { + return lookupEnvOrDefault(CodeFlareTestRayVersion, RayVersion) +} + +func GetRayImage() string { + return lookupEnvOrDefault(CodeFlareTestRayImage, RayImage) +} + +func lookupEnvOrDefault(key, value string) string { + if v, ok := os.LookupEnv(key); ok { + return v } - return controllers.CodeFlareSDKVersion + return value } diff --git a/test/support/defaults.go b/test/support/defaults.go new file mode 100644 index 00000000..a0aaad42 --- /dev/null +++ b/test/support/defaults.go @@ -0,0 +1,11 @@ +package support + +// *********************** +// DO NOT EDIT THIS FILE +// *********************** + +const ( + CodeFlareSDKVersion = "0.4.4" + RayVersion = "2.5.0" + RayImage = "rayproject/ray:2.5.0" +) From 2d477f6fc8d5a287cc3efe6f5133941aad1525af Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 16:40:36 +0200 Subject: [PATCH 023/377] test: Parameterize PyTorch image --- test/e2e/mnist_pytorch_mcad_job_test.go | 2 +- test/support/codeflare.go | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index f1664a53..1b1125af 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -77,7 +77,7 @@ func TestMNISTPyTorchMCAD(t *testing.T) { Containers: []corev1.Container{ { Name: "job", - Image: "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime", + Image: GetPyTorchImage(), Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ { diff --git a/test/support/codeflare.go b/test/support/codeflare.go index 0763bbd3..488a8fff 100644 --- a/test/support/codeflare.go +++ b/test/support/codeflare.go @@ -23,9 +23,10 @@ import ( // The environment variables hereafter can be used to change the components // used for testing. const ( - CodeFlareTestSdkVersion = "CODEFLARE_TEST_SDK_VERSION" - CodeFlareTestRayVersion = "CODEFLARE_TEST_RAY_VERSION" - CodeFlareTestRayImage = "CODEFLARE_TEST_RAY_IMAGE" + CodeFlareTestSdkVersion = "CODEFLARE_TEST_SDK_VERSION" + CodeFlareTestRayVersion = "CODEFLARE_TEST_RAY_VERSION" + CodeFlareTestRayImage = "CODEFLARE_TEST_RAY_IMAGE" + CodeFlareTestPyTorchImage = "CODEFLARE_TEST_PYTORCH_IMAGE" ) func GetCodeFlareSDKVersion() string { @@ -40,6 +41,10 @@ func GetRayImage() string { return lookupEnvOrDefault(CodeFlareTestRayImage, RayImage) } +func GetPyTorchImage() string { + return lookupEnvOrDefault(CodeFlareTestPyTorchImage, "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime") +} + func lookupEnvOrDefault(key, value string) string { if v, ok := os.LookupEnv(key); ok { return v From 57b427d7c88bb4cc45a7ee52933339f26b64472c Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 16:42:48 +0200 Subject: [PATCH 024/377] test: Add FIXME for SDK user base image --- test/e2e/mnist_raycluster_sdk_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 3ba168a0..d919fdcf 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -153,7 +153,9 @@ func TestMNISTRayClusterSDK(t *testing.T) { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "test", + Name: "test", + // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed + // See https://github.com/project-codeflare/codeflare-sdk/pull/146 Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, VolumeMounts: []corev1.VolumeMount{ From 9c79ab0a9d38af1fcadf1a9ed2c05b7a212279cc Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 17:35:38 +0200 Subject: [PATCH 025/377] Align go.mod with MCAD version --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c3ce8665..f70edf1e 100644 --- a/Makefile +++ b/Makefile @@ -214,12 +214,16 @@ vet: ## Run go vet against code. ##@ Build +.PHONY: modules +modules: ## Update Go dependencies. + go get $(MCAD_REPO)@$(MCAD_VERSION) + .PHONY: build -build: defaults generate fmt vet ## Build manager binary. +build: modules defaults generate fmt vet ## Build manager binary. go build -o bin/manager main.go .PHONY: run -run: defaults manifests generate fmt vet ## Run a controller from your host. +run: modules defaults manifests generate fmt vet ## Run a controller from your host. go run ./main.go .PHONY: image-build From f7b7a3a0dcbf804e699223b4889fcddc3846e425 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 28 Jun 2023 15:49:46 +0200 Subject: [PATCH 026/377] test: Print Ray job logs after successful or failed completion --- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 15 ++++++++------- test/support/ray.go | 15 +++++++++++---- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 33a9e6a5..024dee92 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -250,13 +250,14 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) - test.T().Logf("Waiting for RayJob %s/%s to complete successfully", rayJob.Namespace, rayJob.Name) - test.Eventually(RayJob(test, namespace, rayJob.Name), TestTimeoutLong). - Should(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) - - rayJob, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Get(test.Ctx(), rayJob.Name, metav1.GetOptions{}) - test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) + test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). + Should(WithTransform(RayJobStatus, Satisfy(rayv1alpha1.IsJobTerminal))) test.T().Logf("Printing RayJob %s/%s logs", rayJob.Namespace, rayJob.Name) - test.T().Log(GetRayJobLogs(test, rayJob)) + test.T().Log(GetRayJobLogs(test, rayJob.Namespace, rayJob.Name)) + + // Assert the Ray job has completed successfully + test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). + To(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) } diff --git a/test/support/ray.go b/test/support/ray.go index 9e0d22d9..75d77570 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -21,7 +21,6 @@ import ( "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" @@ -29,20 +28,28 @@ import ( const RayJobDefaultClusterSelectorKey = "ray.io/cluster" -func RayJob(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayJob { +func RayJob(t Test, namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayJob { return func(g gomega.Gomega) *rayv1alpha1.RayJob { - job, err := t.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) + job, err := t.Client().Ray().RayV1alpha1().RayJobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) return job } } +func GetRayJob(t Test, namespace, name string) *rayv1alpha1.RayJob { + t.T().Helper() + return RayJob(t, namespace, name)(t) +} + func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus { return job.Status.JobStatus } -func GetRayJobLogs(t Test, job *rayv1alpha1.RayJob) string { +func GetRayJobLogs(t Test, namespace, name string) string { t.T().Helper() + + job := GetRayJob(t, namespace, name) + response := t.Client().Core().CoreV1().RESTClient(). Get(). AbsPath("/api/v1/namespaces", job.Namespace, "services", "http:"+job.Status.RayClusterName+"-head-svc:dashboard", "proxy", "api", "jobs", job.Status.JobId, "logs"). From 321a8aaec4c5239da47b875539caad6484e7acc3 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 29 Jun 2023 11:18:42 +0200 Subject: [PATCH 027/377] test: Upload job logs --- .github/workflows/e2e_tests.yaml | 12 ++++++ test/e2e/mnist_pytorch_mcad_job_test.go | 4 +- test/e2e/mnist_raycluster_sdk_test.go | 4 +- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 4 +- test/e2e/support.go | 1 + test/support/batch.go | 6 +-- test/support/codeflare.go | 9 +++- test/support/core.go | 4 +- test/support/ray.go | 4 +- test/support/test.go | 41 +++++++++++++++++-- test/support/utils.go | 20 +++++++++ 11 files changed, 90 insertions(+), 19 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 1f8263d5..8d9318c7 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -169,6 +169,9 @@ jobs: export CODEFLARE_TEST_TIMEOUT_MEDIUM=3m export CODEFLARE_TEST_TIMEOUT_LONG=8m + export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} + echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV + make test-e2e - name: Print CodeFlare operator logs @@ -188,3 +191,12 @@ jobs: run: | echo "Printing KubeRay operator logs" kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay + + - name: Upload logs + uses: actions/upload-artifact@v3 + if: always() && steps.deploy.outcome == 'success' + with: + name: logs + retention-days: 10 + path: | + ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 1b1125af..07bcf95a 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -151,8 +151,8 @@ func TestMNISTPyTorchMCAD(t *testing.T) { WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), )) - // Print the job logs - PrintJobLogs(test, job.Namespace, job.Name) + // Retrieve the job logs + WriteJobLogs(test, job.Namespace, job.Name) // Assert the job has completed successfully test.Expect(GetJob(test, job.Namespace, job.Name)). diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index d919fdcf..4e804fd5 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -195,8 +195,8 @@ func TestMNISTRayClusterSDK(t *testing.T) { WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), )) - // Print the job logs - PrintJobLogs(test, job.Namespace, job.Name) + // Retrieve the job logs + WriteJobLogs(test, job.Namespace, job.Name) // Assert the job has completed successfully test.Expect(GetJob(test, job.Namespace, job.Name)). diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 024dee92..f20ba0a7 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -254,8 +254,8 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). Should(WithTransform(RayJobStatus, Satisfy(rayv1alpha1.IsJobTerminal))) - test.T().Logf("Printing RayJob %s/%s logs", rayJob.Namespace, rayJob.Name) - test.T().Log(GetRayJobLogs(test, rayJob.Namespace, rayJob.Name)) + test.T().Logf("Retrieving RayJob %s/%s logs", rayJob.Namespace, rayJob.Name) + WriteToOutputDir(test, rayJob.Name, Log, GetRayJobLogs(test, rayJob.Namespace, rayJob.Name)) // Assert the Ray job has completed successfully test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). diff --git a/test/e2e/support.go b/test/e2e/support.go index 7847c075..14bf19ce 100644 --- a/test/e2e/support.go +++ b/test/e2e/support.go @@ -28,6 +28,7 @@ import ( var files embed.FS func ReadFile(t support.Test, fileName string) []byte { + t.T().Helper() file, err := files.ReadFile(fileName) t.Expect(err).NotTo(gomega.HaveOccurred()) return file diff --git a/test/support/batch.go b/test/support/batch.go index 6349efa5..db8feeb5 100644 --- a/test/support/batch.go +++ b/test/support/batch.go @@ -38,7 +38,7 @@ func GetJob(t Test, namespace, name string) *batchv1.Job { return Job(t, namespace, name)(t) } -func PrintJobLogs(t Test, namespace, name string) { +func WriteJobLogs(t Test, namespace, name string) { t.T().Helper() job := GetJob(t, namespace, name) @@ -51,8 +51,8 @@ func PrintJobLogs(t Test, namespace, name string) { t.T().Errorf("Job %s/%s has no pods scheduled", job.Namespace, job.Name) } else { for i, pod := range pods { - t.T().Logf("Printing Pod %s/%s logs", pod.Namespace, pod.Name) - t.T().Log(GetPodLogs(t, &pods[i], corev1.PodLogOptions{})) + t.T().Logf("Retrieving Pod %s/%s logs", pod.Namespace, pod.Name) + WriteToOutputDir(t, pod.Name, Log, GetPodLogs(t, &pods[i], corev1.PodLogOptions{})) } } } diff --git a/test/support/codeflare.go b/test/support/codeflare.go index 488a8fff..04b1f3e9 100644 --- a/test/support/codeflare.go +++ b/test/support/codeflare.go @@ -20,13 +20,18 @@ import ( "os" ) -// The environment variables hereafter can be used to change the components -// used for testing. const ( + // The environment variables hereafter can be used to change the components + // used for testing. + CodeFlareTestSdkVersion = "CODEFLARE_TEST_SDK_VERSION" CodeFlareTestRayVersion = "CODEFLARE_TEST_RAY_VERSION" CodeFlareTestRayImage = "CODEFLARE_TEST_RAY_IMAGE" CodeFlareTestPyTorchImage = "CODEFLARE_TEST_PYTORCH_IMAGE" + + // The testing output directory, to write output files into. + + CodeFlareTestOutputDir = "CODEFLARE_TEST_OUTPUT_DIR" ) func GetCodeFlareSDKVersion() string { diff --git a/test/support/core.go b/test/support/core.go index 273c049a..a9be18bf 100644 --- a/test/support/core.go +++ b/test/support/core.go @@ -43,7 +43,7 @@ func GetPods(t Test, namespace string, options metav1.ListOptions) []corev1.Pod return pods.Items } -func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) string { +func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) []byte { t.T().Helper() stream, err := t.Client().Core().CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetName(), &options).Stream(t.Ctx()) t.Expect(err).NotTo(gomega.HaveOccurred()) @@ -55,5 +55,5 @@ func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) string { bytes, err := io.ReadAll(stream) t.Expect(err).NotTo(gomega.HaveOccurred()) - return string(bytes) + return bytes } diff --git a/test/support/ray.go b/test/support/ray.go index 75d77570..3833d43b 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -45,7 +45,7 @@ func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus { return job.Status.JobStatus } -func GetRayJobLogs(t Test, namespace, name string) string { +func GetRayJobLogs(t Test, namespace, name string) []byte { t.T().Helper() job := GetRayJob(t, namespace, name) @@ -61,5 +61,5 @@ func GetRayJobLogs(t Test, namespace, name string) string { t.Expect(json.Unmarshal(bytes, &body)).To(gomega.Succeed()) t.Expect(body).To(gomega.HaveKey("logs")) - return body["logs"] + return []byte(body["logs"]) } diff --git a/test/support/test.go b/test/support/test.go index 5b3b271b..75556ca4 100644 --- a/test/support/test.go +++ b/test/support/test.go @@ -18,6 +18,8 @@ package support import ( "context" + "os" + "path" "sync" "testing" @@ -30,6 +32,7 @@ type Test interface { T() *testing.T Ctx() context.Context Client() Client + OutputDir() string gomega.Gomega @@ -70,9 +73,13 @@ type T struct { *gomega.WithT t *testing.T // nolint: containedctx - ctx context.Context - client Client - once sync.Once + ctx context.Context + client Client + outputDir string + once struct { + client sync.Once + outputDir sync.Once + } } func (t *T) T() *testing.T { @@ -84,7 +91,8 @@ func (t *T) Ctx() context.Context { } func (t *T) Client() Client { - t.once.Do(func() { + t.T().Helper() + t.once.client.Do(func() { c, err := newTestClient() if err != nil { t.T().Fatalf("Error creating client: %v", err) @@ -94,6 +102,31 @@ func (t *T) Client() Client { return t.client } +func (t *T) OutputDir() string { + t.T().Helper() + t.once.outputDir.Do(func() { + if parent, ok := os.LookupEnv(CodeFlareTestOutputDir); ok { + if !path.IsAbs(parent) { + if cwd, err := os.Getwd(); err == nil { + // best effort to output the parent absolute path + parent = path.Join(cwd, parent) + } + } + t.T().Logf("Creating output directory in parent directory: %s", parent) + dir, err := os.MkdirTemp(parent, t.T().Name()) + if err != nil { + t.T().Fatalf("Error creating output directory: %v", err) + } + t.outputDir = dir + } else { + t.T().Logf("Creating ephemeral output directory as %s env variable is unset", CodeFlareTestOutputDir) + t.outputDir = t.T().TempDir() + } + t.T().Logf("Output directory has been created at: %s", t.outputDir) + }) + return t.outputDir +} + func (t *T) NewTestNamespace(options ...Option[*corev1.Namespace]) *corev1.Namespace { t.T().Helper() namespace := createTestNamespace(t, options...) diff --git a/test/support/utils.go b/test/support/utils.go index ed40309f..595ac410 100644 --- a/test/support/utils.go +++ b/test/support/utils.go @@ -16,6 +16,26 @@ limitations under the License. package support +import ( + "io/fs" + "os" + "path" + + "github.com/onsi/gomega" +) + func Ptr[T any](v T) *T { return &v } + +type OutputType string + +const ( + Log OutputType = "log" +) + +func WriteToOutputDir(t Test, fileName string, fileType OutputType, data []byte) { + t.T().Helper() + t.Expect(os.WriteFile(path.Join(t.OutputDir(), fileName+"."+string(fileType)), data, fs.ModePerm)). + To(gomega.Succeed()) +} From 630199b0fd11685c6ea23fe6ea705e25dd913229 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 29 Jun 2023 13:35:05 +0200 Subject: [PATCH 028/377] test Remove unused functions --- test/support/gomega.go | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 test/support/gomega.go diff --git a/test/support/gomega.go b/test/support/gomega.go deleted file mode 100644 index 74897502..00000000 --- a/test/support/gomega.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - "github.com/onsi/gomega/gstruct" - "github.com/onsi/gomega/types" -) - -func EqualP(expected interface{}) types.GomegaMatcher { - return gstruct.PointTo(gomega.Equal(expected)) -} - -func MatchFieldsP(options gstruct.Options, fields gstruct.Fields) types.GomegaMatcher { - return gstruct.PointTo(gstruct.MatchFields(options, fields)) -} From 9771055b8d28c1c3e51e0039a68abe6791665e19 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 30 Jun 2023 14:56:33 +0200 Subject: [PATCH 029/377] test: Fix Unexpected kind-action input --- .github/workflows/e2e_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 8d9318c7..2783c52b 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -90,7 +90,7 @@ jobs: - name: Setup KinD cluster uses: helm/kind-action@v1.5.0 with: - name: cluster + cluster_name: cluster version: v0.17.0 config: ${{ env.KIND_CONFIG_FILE }} From 06d47ed34464e6ac963ef9ee33f078015b5caf86 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 30 Jun 2023 15:14:18 +0200 Subject: [PATCH 030/377] test: Format test output using gotestfmt --- .github/workflows/e2e_tests.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 2783c52b..1ec1cd0a 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -68,6 +68,11 @@ jobs: with: go-version: v1.18 + - name: Set up gotestfmt + uses: gotesttools/gotestfmt-action@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Container image registry run: | podman run -d -p 5000:5000 --name registry registry:2.8.1 @@ -172,7 +177,8 @@ jobs: export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV - make test-e2e + set -euo pipefail + go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Print CodeFlare operator logs if: always() && steps.deploy.outcome == 'success' From d665dcc325db0e4a0848e8576cd9876222ed8ae4 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 30 Jun 2023 15:52:00 +0200 Subject: [PATCH 031/377] test: Add codeflare stack logs to uploaded artifacts --- .github/workflows/e2e_tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 1ec1cd0a..0c6bb2b9 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -184,19 +184,19 @@ jobs: if: always() && steps.deploy.outcome == 'success' run: | echo "Printing CodeFlare operator logs" - kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator + kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log - name: Print MCAD controller logs if: always() && steps.deploy.outcome == 'success' run: | echo "Printing MCAD controller logs" - kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher + kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log - name: Print KubeRay operator logs if: always() && steps.deploy.outcome == 'success' run: | echo "Printing KubeRay operator logs" - kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay + kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log - name: Upload logs uses: actions/upload-artifact@v3 From 9d536567b126006f5c7496f970cb780241a6eae4 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 30 Jun 2023 16:44:06 +0200 Subject: [PATCH 032/377] test: Add description to e2e tests --- test/e2e/mnist_pytorch_mcad_job_test.go | 1 + test/e2e/mnist_raycluster_sdk_test.go | 6 ++++++ test/e2e/mnist_rayjob_mcad_raycluster_test.go | 2 ++ 3 files changed, 9 insertions(+) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 07bcf95a..41376a0b 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -30,6 +30,7 @@ import ( mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) +// Trains the MNIST dataset as a batch Job managed by MCAD, and asserts successful completion of the training job. func TestMNISTPyTorchMCAD(t *testing.T) { test := With(t) test.T().Parallel() diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 4e804fd5..202f3bcc 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -32,6 +32,12 @@ import ( mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) +// Creates a Ray cluster, and trains the MNIST dataset using the CodeFlare SDK. +// Asserts successful completion of the training job. +// +// This covers the installation of the CodeFlare SDK, as well as the RBAC required +// for the SDK to successfully perform requests to the cluster, on behalf of the +// impersonated user. func TestMNISTRayClusterSDK(t *testing.T) { test := With(t) test.T().Parallel() diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index f20ba0a7..b6b01653 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -31,6 +31,8 @@ import ( rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" ) +// Trains the MNIST dataset as a RayJob, executed by a Ray cluster managed by MCAD, +// and asserts successful completion of the training job. func TestMNISTRayJobMCADRayCluster(t *testing.T) { test := With(t) test.T().Parallel() From 99b8e7c6e9f86a37406291952d30f6a29c61a3ae Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 30 Jun 2023 17:11:26 +0200 Subject: [PATCH 033/377] test: Factorize e2e tests setup --- .github/workflows/e2e_tests.yaml | 55 +----------------------- Makefile | 6 +-- test/e2e/setup.sh | 73 ++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 56 deletions(-) create mode 100755 test/e2e/setup.sh diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 0c6bb2b9..435c72f0 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -108,65 +108,14 @@ jobs: - name: Deploy CodeFlare stack id: deploy run: | - KUBERAY_VERSION=$(make get-kuberay-version) - echo Deploying KubeRay ${KUBERAY_VERSION} - kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=90s" - echo Deploying CodeFlare operator IMG="${REGISTRY_ADDRESS}"/codeflare-operator make image-push -e IMG="${IMG}" make deploy -e IMG="${IMG}" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager - echo Deploying MCAD controller - kubectl create ns codeflare-system - cat < Date: Fri, 30 Jun 2023 17:16:07 +0200 Subject: [PATCH 034/377] test: Update e2e tests local run documentation --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 31f21640..31ce40f0 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ CodeFlare Stack Compatibility Matrix The e2e tests can be executed locally by running the following commands: -1. Setup the test cluster: +1. Use an existing cluster, or set up a test cluster, e.g.: ```bash # Create a KinD cluster @@ -30,13 +30,20 @@ The e2e tests can be executed locally by running the following commands: $ make install ``` -2. Start the operator locally: +2. Set up the CodeFlare stack: + ```bash + $ make setup-e2e + ``` + +3. Start the operator locally: ```bash $ make run ``` -3. In a separate terminal, run the e2e suite: + Alternatively, You can run the operator from your IDE / debugger. + +4. In a separate terminal, run the e2e suite: ```bash $ make test-e2e From 8c1831ae50d2d56ad19e3d0b1c36b2b10c67b385 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 5 Jul 2023 14:30:52 +0200 Subject: [PATCH 035/377] test: Write logs also for jobs that have timed out --- test/e2e/mnist_pytorch_mcad_job_test.go | 6 +++--- test/e2e/mnist_raycluster_sdk_test.go | 6 +++--- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 6 +++--- test/support/ray.go | 5 +++++ 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 41376a0b..168c7b13 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -145,6 +145,9 @@ func TestMNISTPyTorchMCAD(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + // Retrieving the job logs once it has completed or timed out + defer WriteJobLogs(test, job.Namespace, job.Name) + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( Or( @@ -152,9 +155,6 @@ func TestMNISTPyTorchMCAD(t *testing.T) { WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), )) - // Retrieve the job logs - WriteJobLogs(test, job.Namespace, job.Name) - // Assert the job has completed successfully test.Expect(GetJob(test, job.Namespace, job.Name)). To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 202f3bcc..e8bb1f5c 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -194,6 +194,9 @@ func TestMNISTRayClusterSDK(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + // Retrieving the job logs once it has completed or timed out + defer WriteJobLogs(test, job.Namespace, job.Name) + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( Or( @@ -201,9 +204,6 @@ func TestMNISTRayClusterSDK(t *testing.T) { WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), )) - // Retrieve the job logs - WriteJobLogs(test, job.Namespace, job.Name) - // Assert the job has completed successfully test.Expect(GetJob(test, job.Namespace, job.Name)). To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index b6b01653..958b3047 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -252,13 +252,13 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) + // Retrieving the job logs once it has completed or timed out + defer WriteRayJobLogs(test, rayJob.Namespace, rayJob.Name) + test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). Should(WithTransform(RayJobStatus, Satisfy(rayv1alpha1.IsJobTerminal))) - test.T().Logf("Retrieving RayJob %s/%s logs", rayJob.Namespace, rayJob.Name) - WriteToOutputDir(test, rayJob.Name, Log, GetRayJobLogs(test, rayJob.Namespace, rayJob.Name)) - // Assert the Ray job has completed successfully test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). To(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) diff --git a/test/support/ray.go b/test/support/ray.go index 3833d43b..cd5a9d87 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -63,3 +63,8 @@ func GetRayJobLogs(t Test, namespace, name string) []byte { return []byte(body["logs"]) } + +func WriteRayJobLogs(t Test, namespace, name string) { + t.T().Logf("Retrieving RayJob %s/%s logs", namespace, name) + WriteToOutputDir(t, name, Log, GetRayJobLogs(t, namespace, name)) +} From 2fa3f901d2f82efec476a85fa2a96ce8746f89dd Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 5 Jul 2023 16:35:07 +0200 Subject: [PATCH 036/377] doc: The operator should be started before setting up e2e tests --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 31ce40f0..5984017e 100644 --- a/README.md +++ b/README.md @@ -30,12 +30,7 @@ The e2e tests can be executed locally by running the following commands: $ make install ``` -2. Set up the CodeFlare stack: - ```bash - $ make setup-e2e - ``` - -3. Start the operator locally: +2. Start the operator locally: ```bash $ make run @@ -43,6 +38,12 @@ The e2e tests can be executed locally by running the following commands: Alternatively, You can run the operator from your IDE / debugger. +3. Set up the test CodeFlare stack: + + ```bash + $ make setup-e2e + ``` + 4. In a separate terminal, run the e2e suite: ```bash From e04fbfa87c852c2788b5be389fd52834279075a2 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 27 Jun 2023 10:09:48 +0200 Subject: [PATCH 037/377] fix: Update MCAD version when installing CRDs --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index b4844103..ef3a05be 100644 --- a/Makefile +++ b/Makefile @@ -242,7 +242,9 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) $(KUSTOMIZE) build config/crd | kubectl apply -f - + git restore config/* .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. From 92c616dcc3cc7c198303988cacb540aa75b6d807 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 7 Jul 2023 11:09:14 +0200 Subject: [PATCH 038/377] Use CodeFlare machine user to push PR into OpenShift community operators repository --- .github/workflows/tag-and-build.yml | 8 ++++---- Makefile | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 413ccf2d..b823fc11 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -94,15 +94,15 @@ jobs: - name: Build bundle and create PR in OpenShift community operators repository run: | - git config --global user.email "codeflare-ci@redhat.com" - git config --global user.name "CodeFlare CI" + git config --global user.email "138894154+codeflare-machine-account@users.noreply.github.com" + git config --global user.name "codeflare-machine-account" make openshift-community-operator-release env: VERSION: ${{ github.event.inputs.version }} PREVIOUS_VERSION: ${{ github.event.inputs.replaces }} INSTASCALE_VERSION: ${{ github.event.inputs.instascale-version }} MCAD_VERSION: ${{ github.event.inputs.mcad-version }} - GH_TOKEN: ${{ secrets.GH_PAT }} + GH_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} IMAGE_ORG_BASE: quay.io/${{ github.event.inputs.quay-organization }} OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} @@ -125,7 +125,7 @@ jobs: uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} - file_pattern: 'README.md controllers/defaults.go *.yaml *.tmpl Makefile' + file_pattern: 'README.md controllers/defaults.go *.yaml Makefile' - name: Creates a release in GitHub run: | diff --git a/Makefile b/Makefile index ef3a05be..f1ff2de9 100644 --- a/Makefile +++ b/Makefile @@ -368,11 +368,11 @@ bundle-push: ## Push the bundle image. .PHONY: openshift-community-operator-release openshift-community-operator-release: install-gh-cli bundle ## build bundle and create PR in OpenShift community operators repository - git clone https://$(GH_TOKEN)@github.com/$(OPERATORS_REPO_FORK_ORG)/community-operators-prod.git + git clone https://x-access-token:$(GH_TOKEN)@github.com/$(OPERATORS_REPO_FORK_ORG)/community-operators-prod.git cd community-operators-prod && git remote add upstream https://github.com/$(OPERATORS_REPO_ORG)/community-operators-prod.git && git pull upstream main && git push origin main cp -r bundle community-operators-prod/operators/codeflare-operator/$(BUNDLE_VERSION) - cd community-operators-prod && git checkout -b codeflare-release-$(BUNDLE_VERSION) && git add operators/codeflare-operator/$(BUNDLE_VERSION)/* && git commit -m "add bundle manifests codeflare version $(BUNDLE_VERSION)" && git push origin codeflare-release-$(BUNDLE_VERSION) - gh pr create --repo $(OPERATORS_REPO_FORK_ORG)/community-operators-prod --title "CodeFlare $(BUNDLE_VERSION)" --body "New release of codeflare operator" --head $(OPERATORS_REPO_ORG):codeflare-release-$(BUNDLE_VERSION) --base main + cd community-operators-prod && git checkout -b codeflare-release-$(BUNDLE_VERSION) && git add operators/codeflare-operator/$(BUNDLE_VERSION)/* && git commit -m "add bundle manifests codeflare version $(BUNDLE_VERSION)" --signoff && git push origin codeflare-release-$(BUNDLE_VERSION) + gh pr create --repo $(OPERATORS_REPO_ORG)/community-operators-prod --title "CodeFlare $(BUNDLE_VERSION)" --body "New release of codeflare operator" --head $(OPERATORS_REPO_FORK_ORG):codeflare-release-$(BUNDLE_VERSION) --base main rm -rf community-operators-prod .PHONY: opm From 644d27bb9146049fa5e65b61bcb9507c473dcf1b Mon Sep 17 00:00:00 2001 From: anishasthana Date: Fri, 7 Jul 2023 16:00:26 +0000 Subject: [PATCH 039/377] Update dependency versions for release v0.0.5 --- Makefile | 4 ++-- README.md | 8 ++++---- controllers/defaults.go | 4 ++-- .../instascale_test_results/case_1/deployment.yaml | 2 +- .../instascale_test_results/case_2/deployment.yaml | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index f1ff2de9..c0037f0e 100644 --- a/Makefile +++ b/Makefile @@ -12,10 +12,10 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.0.4 +INSTASCALE_VERSION ?= v0.0.5 # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.31.0 +MCAD_VERSION ?= v1.32.0 # MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources MCAD_REF ?= release-${MCAD_VERSION} MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher diff --git a/README.md b/README.md index 5984017e..cacc7019 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------| -| CodeFlare Operator | v0.0.4 | -| Multi-Cluster App Dispatcher | v1.31.0 | -| CodeFlare-SDK | v0.4.4 | -| InstaScale | v0.0.4 | +| CodeFlare Operator | v0.0.5 | +| Multi-Cluster App Dispatcher | v1.32.0 | +| CodeFlare-SDK | v0.5.0 | +| InstaScale | v0.0.5 | | KubeRay | v0.5.0 | diff --git a/controllers/defaults.go b/controllers/defaults.go index d3cac3d0..4dc0b777 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,6 +5,6 @@ package controllers // *********************** const ( - MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.31.0" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.4" + MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.32.0" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.5" ) diff --git a/controllers/testdata/instascale_test_results/case_1/deployment.yaml b/controllers/testdata/instascale_test_results/case_1/deployment.yaml index 31c3896f..bb398dbb 100644 --- a/controllers/testdata/instascale_test_results/case_1/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_1/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.4 + image: quay.io/project-codeflare/instascale-controller:v0.0.5 resources: limits: cpu: '2' diff --git a/controllers/testdata/instascale_test_results/case_2/deployment.yaml b/controllers/testdata/instascale_test_results/case_2/deployment.yaml index eb259ec8..ef7b8da1 100644 --- a/controllers/testdata/instascale_test_results/case_2/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_2/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.4 + image: quay.io/project-codeflare/instascale-controller:v0.0.5 resources: limits: cpu: '1' From 598f32b1d03c7e3df8a5642d2ec311f83c4dc203 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Sun, 9 Jul 2023 00:42:11 -0400 Subject: [PATCH 040/377] Fix mcad tag reference for CRDs Upstream is no longer publishing release branches of the form `release-vX.Y.Z` Instead, it is publishing tags of the form `vX.Y.Z`, this commit makes it so Signed-off-by: Anish Asthana --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c0037f0e..5eb47e4c 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,9 @@ MCAD_VERSION ?= v1.32.0 # MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources MCAD_REF ?= release-${MCAD_VERSION} MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher -MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_REF} +# Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) +# The image is still published using the MCAD_REF format (i.e release-vX.Y.Z) +MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) KUBERAY_VERSION ?= v0.5.0 From cd43ff56c4413984e17b64bae9ad142ee10be919 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 10 Jul 2023 13:21:00 +0200 Subject: [PATCH 041/377] Adjust Makefile versions before building operator image --- .github/workflows/tag-and-build.yml | 36 +++++++++++++++-------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index b823fc11..5f07893e 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -49,6 +49,9 @@ jobs: permissions: contents: write + env: + IMAGE_ORG_BASE: quay.io/${{ github.event.inputs.quay-organization }} + steps: - uses: actions/checkout@v3 @@ -73,6 +76,20 @@ jobs: - name: Install operator-sdk run: make install-operator-sdk + - name: Adjust Compatibility Matrix in readme + run: | + sed -i -E "s/(.*CodeFlare Operator.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.version }}\2/" README.md + sed -i -E "s/(.*Multi-Cluster App Dispatcher.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" README.md + sed -i -E "s/(.*CodeFlare-SDK.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.codeflare-sdk-version }}\2/" README.md + sed -i -E "s/(.*InstaScale.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" README.md + + - name: Adjust MCAD and InstaScale dependencies in the code + run: | + sed -i -E "s/(.*MCAD_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" Makefile + sed -i -E "s/(.*INSTASCALE_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" Makefile + sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_1/deployment.yaml + sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_2/deployment.yaml + - name: Login to Quay.io uses: redhat-actions/podman-login@v1 with: @@ -103,29 +120,14 @@ jobs: INSTASCALE_VERSION: ${{ github.event.inputs.instascale-version }} MCAD_VERSION: ${{ github.event.inputs.mcad-version }} GH_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - IMAGE_ORG_BASE: quay.io/${{ github.event.inputs.quay-organization }} OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} - - name: Adjust Compatibility Matrix in readme - run: | - sed -i -E "s/(.*CodeFlare Operator.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.version }}\2/" README.md - sed -i -E "s/(.*Multi-Cluster App Dispatcher.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" README.md - sed -i -E "s/(.*CodeFlare-SDK.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.codeflare-sdk-version }}\2/" README.md - sed -i -E "s/(.*InstaScale.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" README.md - - - name: Adjust MCAD and InstaScale dependencies in the code - run: | - sed -i -E "s/(.*MCAD_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" Makefile - sed -i -E "s/(.*INSTASCALE_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" Makefile - sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_1/deployment.yaml - sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_2/deployment.yaml - - - name: Commit readme changes back to repository + - name: Commit changes in the code back to repository uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} - file_pattern: 'README.md controllers/defaults.go *.yaml Makefile' + file_pattern: 'README.md controllers/defaults.go *.yaml Makefile go.mod go.sum' - name: Creates a release in GitHub run: | From 1380bcc093874404c2950e737a0a9a447e6bb20a Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 10 Jul 2023 15:56:32 +0100 Subject: [PATCH 042/377] updating rbacs --- config/rbac/role.yaml | 7 +++++++ controllers/instascale_controller.go | 1 + 2 files changed, 8 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ab9bc905..d0c21ba6 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -96,6 +96,13 @@ rules: - get - patch - update +- apiGroups: + - config.openshift.io + resources: + - clusterversions + verbs: + - get + - list - apiGroups: - coordination.k8s.io resources: diff --git a/controllers/instascale_controller.go b/controllers/instascale_controller.go index 31fcb53b..80e38a53 100644 --- a/controllers/instascale_controller.go +++ b/controllers/instascale_controller.go @@ -90,6 +90,7 @@ func (r *InstaScaleReconciler) Apply(owner mf.Owner, params *InstaScaleParams, t // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings,verbs=get;list;watch;create;update;delete // +kubebuilder:rbac:groups=machine.openshift.io,resources=*,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=mcad.ibm.com,resources=appwrappers;queuejobs;schedulingspecs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=config.openshift.io,resources=clusterversions,verbs=get;list // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. From d376f12c321cbd70ab9f3718656495f17cf3c717 Mon Sep 17 00:00:00 2001 From: anishasthana Date: Mon, 10 Jul 2023 16:37:28 +0000 Subject: [PATCH 043/377] Update dependency versions for release v0.0.6 --- README.md | 2 +- go.mod | 2 +- go.sum | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cacc7019..e5f25443 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------| -| CodeFlare Operator | v0.0.5 | +| CodeFlare Operator | v0.0.6 | | Multi-Cluster App Dispatcher | v1.32.0 | | CodeFlare-SDK | v0.5.0 | | InstaScale | v0.0.5 | diff --git a/go.mod b/go.mod index 621853fb..7d8c01c8 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/manifestival/manifestival v0.7.2 github.com/onsi/ginkgo/v2 v2.9.2 github.com/onsi/gomega v1.27.6 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 diff --git a/go.sum b/go.sum index 8e877d1f..f16a6996 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= @@ -55,6 +56,7 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -79,6 +81,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -97,6 +100,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -313,9 +317,12 @@ github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:Fecb github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -330,6 +337,7 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -382,6 +390,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09 github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -400,6 +409,7 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -430,6 +440,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0 h1:vq4fAuvlv4Zvnx0dA53WaYkTWG1BFxAkamxuzHfZO2M= github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0/go.mod h1:fmbU5LuV1Z2Sbu1FCEoVuw8qxDFcalXvkPyMfGZHHTc= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 h1:17lwqvxcWBTi9245lVZgJPvksGwAGA58s8dl7PzpbyE= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0/go.mod h1:fmbU5LuV1Z2Sbu1FCEoVuw8qxDFcalXvkPyMfGZHHTc= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -471,7 +483,9 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -499,6 +513,7 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= @@ -511,7 +526,9 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -905,6 +922,7 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 70f0719289e5eed135827d65b3ea40af44121442 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 11 Jul 2023 09:46:23 +0200 Subject: [PATCH 044/377] Run go mod tidy before comitting files in release workflow --- .github/workflows/tag-and-build.yml | 4 ++++ go.sum | 18 ------------------ 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 5f07893e..339d5b6f 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -123,6 +123,10 @@ jobs: OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} + - name: Cleanup the go.mod and go.sum + run: | + go mod tidy + - name: Commit changes in the code back to repository uses: stefanzweifel/git-auto-commit-action@v4 with: diff --git a/go.sum b/go.sum index f16a6996..c143ff7a 100644 --- a/go.sum +++ b/go.sum @@ -33,7 +33,6 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= @@ -56,7 +55,6 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -81,7 +79,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -100,7 +97,6 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -317,12 +313,9 @@ github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:Fecb github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -337,7 +330,6 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -390,7 +382,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09 github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -409,7 +400,6 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -438,8 +428,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0 h1:vq4fAuvlv4Zvnx0dA53WaYkTWG1BFxAkamxuzHfZO2M= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.31.0/go.mod h1:fmbU5LuV1Z2Sbu1FCEoVuw8qxDFcalXvkPyMfGZHHTc= github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 h1:17lwqvxcWBTi9245lVZgJPvksGwAGA58s8dl7PzpbyE= github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0/go.mod h1:fmbU5LuV1Z2Sbu1FCEoVuw8qxDFcalXvkPyMfGZHHTc= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -483,9 +471,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -513,7 +499,6 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= @@ -526,9 +511,7 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -922,7 +905,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 9c269d0881a0a37166c120963d747b543413df4e Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 15 Jun 2023 14:52:53 +0200 Subject: [PATCH 045/377] OLM install and upgrade PR check for CodeFlare stack --- .github/actions/kind/action.yml | 49 +++++ .../e2e => .github/resources-kind}/kind.yaml | 0 .../resources-olm-upgrade/catalogsource.yaml | 12 ++ .../resources-olm-upgrade/operatorgroup.yaml | 5 + .../resources-olm-upgrade/subscription.yaml | 11 + .github/workflows/e2e_tests.yaml | 46 +---- .github/workflows/olm_tests.yaml | 190 ++++++++++++++++++ Makefile | 37 +++- 8 files changed, 305 insertions(+), 45 deletions(-) create mode 100644 .github/actions/kind/action.yml rename {test/e2e => .github/resources-kind}/kind.yaml (100%) create mode 100644 .github/resources-olm-upgrade/catalogsource.yaml create mode 100644 .github/resources-olm-upgrade/operatorgroup.yaml create mode 100644 .github/resources-olm-upgrade/subscription.yaml create mode 100644 .github/workflows/olm_tests.yaml diff --git a/.github/actions/kind/action.yml b/.github/actions/kind/action.yml new file mode 100644 index 00000000..58122e8a --- /dev/null +++ b/.github/actions/kind/action.yml @@ -0,0 +1,49 @@ +name: "Set up KinD" +description: "Step to start and configure KinD cluster" + +runs: + using: "composite" + steps: + - name: Init directories + shell: bash + run: | + TEMP_DIR="$(pwd)/tmp" + mkdir -p "${TEMP_DIR}" + echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV + + mkdir -p "$(pwd)/bin" + echo "$(pwd)/bin" >> $GITHUB_PATH + + - name: Container image registry + shell: bash + run: | + podman run -d -p 5000:5000 --name registry registry:2.8.1 + + export REGISTRY_ADDRESS=$(hostname -i):5000 + echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV + echo "Container image registry started at ${REGISTRY_ADDRESS}" + + KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml + echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV + envsubst < .github/resources-kind/kind.yaml > ${KIND_CONFIG_FILE} + + sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <> $GITHUB_ENV - - mkdir -p "$(pwd)/bin" - echo "$(pwd)/bin" >> $GITHUB_PATH - - name: Set Go uses: actions/setup-go@v3 with: @@ -73,37 +64,8 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - - name: Container image registry - run: | - podman run -d -p 5000:5000 --name registry registry:2.8.1 - - export REGISTRY_ADDRESS=$(hostname -i):5000 - echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV - echo "Container image registry started at ${REGISTRY_ADDRESS}" - - KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml - echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV - envsubst < ./test/e2e/kind.yaml > ${KIND_CONFIG_FILE} - - sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <> $GITHUB_ENV diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml new file mode 100644 index 00000000..9e1061c9 --- /dev/null +++ b/.github/workflows/olm_tests.yaml @@ -0,0 +1,190 @@ +# This workflow will build the CodeFlare Operator image and catalog containing bundle with this image, execute OLM upgrade tests using this catalog + +name: OLM Install and Upgrade + +on: + pull_request: + branches: + - main + - 'release-*' + paths-ignore: + - 'docs/**' + - '**.adoc' + - '**.md' + - 'LICENSE' + +concurrency: + group: ${{ github.head_ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + kubernetes: + runs-on: ubuntu-20.04 + timeout-minutes: 60 + env: + OLM_VERSION: v0.24.0 + VERSION: "v0.0.0-ghaction" # Need to supply some semver version for bundle to be properly generated + CATALOG_BASE_IMG: "registry.access.redhat.com/redhat/community-operator-index:v4.13" + CODEFLARE_TEST_TIMEOUT_SHORT: "1m" + CODEFLARE_TEST_TIMEOUT_MEDIUM: "5m" + CODEFLARE_TEST_TIMEOUT_LONG: "10m" + + steps: + - name: Cleanup + run: | + ls -lart + echo "Initial status:" + df -h + + echo "Cleaning up resources:" + sudo swapoff -a + sudo rm -f /swapfile + sudo apt clean + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + docker rmi $(docker image ls -aq) + + echo "Final status:" + df -h + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # fetching also previous commits to get tags + + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.18 + + - name: Set up gotestfmt + uses: gotesttools/gotestfmt-action@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup and start KinD cluster + uses: ./.github/actions/kind + + - name: Install OLM + run: | + kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml + # wait for a while to be sure CRDs are installed + sleep 1 + kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/olm.yaml + + - name: Create openshift-operator namespace and OperatorGroup + run: | + # Need to use openshift-operator namespace due to https://github.com/project-codeflare/codeflare-operator/issues/161 + kubectl create namespace openshift-operators + kubectl create -f .github/resources-olm-upgrade/operatorgroup.yaml + + - name: Deploy latest released CodeFlare operator from OLM + id: deploy + run: | + echo Deploying CodeFlare operator using Subscription + envsubst < .github/resources-olm-upgrade/catalogsource.yaml > ${{ env.TEMP_DIR }}/catalogsource.yaml + envsubst < .github/resources-olm-upgrade/subscription.yaml > ${{ env.TEMP_DIR }}/subscription.yaml + + kubectl create -f ${{ env.TEMP_DIR }}/catalogsource.yaml + make wait-for-catalog-source + + kubectl create -f ${{ env.TEMP_DIR }}/subscription.yaml + + echo Waiting for Subscription to be ready + make wait-for-subscription + + echo Waiting for Deployment to be ready + make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators" + env: + CATALOG_SOURCE_NAME: "codeflare-olm-test" + CATALOG_SOURCE_NAMESPACE: "olm" + SUBSCRIPTION_NAME: "codeflare-operator" + SUBSCRIPTION_NAMESPACE: "openshift-operators" + + - name: Store latest CSV version as PREVIOUS_VERSION env variable (used for bundle build) + run: | + CSV_VERSION=$(kubectl get ClusterServiceVersion -l operators.coreos.com/codeflare-operator.openshift-operators='' -n openshift-operators -o json | jq -r .items[].spec.version) + echo "PREVIOUS_VERSION=v$CSV_VERSION" >> $GITHUB_ENV + + - name: Deploy CodeFlare stack (MCAD, KubeRay) + run: | + make setup-e2e + + - name: Build operator and catalog image + run: | + make image-push + make bundle-build + make bundle-push + make catalog-build-from-index + make catalog-push + env: + IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator:v0.0.1" + BUNDLE_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-bundle:v0.0.1" + CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1" + OPM_BUNDLE_OPT: "--use-http" + BUNDLE_PUSH_OPT: "--tls-verify=false" + CATALOG_PUSH_OPT: "--tls-verify=false" + + - name: Update Operator to the built version + run: | + ORIGINAL_POD_NAME=$(kubectl get pod -l app.kubernetes.io/name=codeflare-operator -n openshift-operators -o json | jq -r .items[].metadata.name) + echo "Running old operator pod name is ${ORIGINAL_POD_NAME}" + + echo Updating custom CatalogSource image to the built CatalogSource with latest operator + kubectl patch CatalogSource codeflare-olm-test -n olm --type merge --patch "{\"spec\":{\"image\":\"${CATALOG_IMG}\"}}" + + echo Waiting for previous operator pod to get deleted + kubectl wait --timeout=120s --for=delete pod/${ORIGINAL_POD_NAME} -n openshift-operators + + echo Waiting for Subscription to be ready + make wait-for-subscription + + echo Waiting for Deployment to be ready + make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators" + + echo Checking that correct CSV is available + CSV_VERSION=$(kubectl get ClusterServiceVersion/codeflare-operator.${VERSION} -n openshift-operators -o json | jq -r .spec.version) + if [ "v${CSV_VERSION}" != "${VERSION}" ]; then + echo "CSV version v${CSV_VERSION} doesn't match expected version ${VERSION}" + exit 1 + fi + env: + CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1" + SUBSCRIPTION_NAME: "codeflare-operator" + SUBSCRIPTION_NAMESPACE: "openshift-operators" + + - name: Run e2e tests against built operator + run: | + export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} + echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV + + set -euo pipefail + go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + + - name: Print CodeFlare operator logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing CodeFlare operator logs" + kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log + + - name: Print MCAD controller logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing MCAD controller logs" + kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log + + - name: Print KubeRay operator logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing KubeRay operator logs" + kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log + + - name: Upload logs + uses: actions/upload-artifact@v3 + if: always() && steps.deploy.outcome == 'success' + with: + name: logs + retention-days: 10 + path: | + ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log diff --git a/Makefile b/Makefile index 5eb47e4c..f10a8735 100644 --- a/Makefile +++ b/Makefile @@ -342,7 +342,7 @@ OPERATOR_SDK_DL_URL := https://github.com/operator-framework/operator-sdk/releas .PHONY: install-operator-sdk install-operator-sdk: $(OPERATOR_SDK) ## Download fixed version operator-sdk binary for consist outcome $(OPERATOR_SDK): $(LOCALBIN) - curl -L $(OPERATOR_SDK_DL_URL)/operator-sdk_$(shell go env GOOS)_$(shell go env GOARCH) --output-dir $(LOCALBIN) --output operator-sdk + curl -L $(OPERATOR_SDK_DL_URL)/operator-sdk_$(shell go env GOOS)_$(shell go env GOARCH) --output $(LOCALBIN)/operator-sdk chmod +x $(OPERATOR_SDK) .PHONY: validate-bundle @@ -366,7 +366,7 @@ bundle-build: bundle ## Build the bundle image. .PHONY: bundle-push bundle-push: ## Push the bundle image. - $(MAKE) image-push IMG=$(BUNDLE_IMG) + podman push $(BUNDLE_IMG) $(BUNDLE_PUSH_OPT) .PHONY: openshift-community-operator-release openshift-community-operator-release: install-gh-cli bundle ## build bundle and create PR in OpenShift community operators repository @@ -413,10 +413,21 @@ endif catalog-build: opm ## Build a catalog image. $(OPM) index add --container-tool podman --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) +# Build a catalog image by adding bundle images to existing catalog using the operator package manager tool, 'opm'. +.PHONY: catalog-build-from-index +catalog-build-from-index: opm ## Build a catalog image. + mkdir catalog + $(OPM) render $(CATALOG_BASE_IMG) -o yaml > catalog/bundles.yaml + $(OPM) render $(BUNDLE_IMG) $(OPM_BUNDLE_OPT) > catalog/codeflare-operator-bundle.yaml + sed -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml + $(OPM) validate catalog + $(OPM) generate dockerfile catalog + podman build . -f catalog.Dockerfile -t $(CATALOG_IMG) + # Push the catalog image. .PHONY: catalog-push catalog-push: ## Push a catalog image. - $(MAKE) image-push IMG=$(CATALOG_IMG) + podman push $(CATALOG_IMG) $(CATALOG_PUSH_OPT) .PHONY: test-unit test-unit: defaults manifests generate fmt vet envtest ## Run unit tests. @@ -429,3 +440,23 @@ test-e2e: defaults manifests generate fmt vet ## Run e2e tests. .PHONY: setup-e2e setup-e2e: ## Set up e2e tests. KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh + +# Wait until CatalogSource is in "READY" state +.PHONY: wait-for-catalog-source +wait-for-catalog-source: + timeout 120 bash -c 'while [[ "$$(kubectl get catalogsource/'$(CATALOG_SOURCE_NAME)' -n '$(CATALOG_SOURCE_NAMESPACE)' -o json | jq -r .status.connectionState.lastObservedState)" != "READY" ]]; do sleep 5 && echo "$$(kubectl get catalogsource/'$(CATALOG_SOURCE_NAME)' -n '$(CATALOG_SOURCE_NAMESPACE)' -o json | jq -r .status.connectionState.lastObservedState)" ; done' + +# Wait until Subscription is in "AtLatestKnown" state +.PHONY: wait-for-subscription +wait-for-subscription: + timeout 300 bash -c 'while [[ "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" != "AtLatestKnown" ]]; do sleep 5 && echo "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" ; done' + +# Default timeout for waiting actions +TIMEOUT ?= 180 + +# Wait until Deployment is in "Available" state +.PHONY: wait-for-deployment +wait-for-deployment: + # Wait until Deployment exists first, then use kubectl wait + timeout $(TIMEOUT) bash -c 'until [[ $$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)') ]]; do sleep 5 && echo "$$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)')"; done' + kubectl wait --timeout=$(TIMEOUT)s --for=condition=Available=true deployment/$(DEPLOYMENT_NAME) -n $(DEPLOYMENT_NAMESPACE) From 0a84b1d7f027fbe431532242a296d4b106b49623 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 18 Jul 2023 15:50:52 +0100 Subject: [PATCH 046/377] Expose CodeFlare Operator metrics endpoint --- config/default/manager_auth_proxy_patch.yaml | 2 +- config/rbac/auth_proxy_service.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 24d5b119..31e1a364 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -35,5 +35,5 @@ spec: - name: manager args: - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" + - "--metrics-bind-address=0.0.0.0:8080" - "--leader-elect" diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml index 25010042..c53c92d1 100644 --- a/config/rbac/auth_proxy_service.yaml +++ b/config/rbac/auth_proxy_service.yaml @@ -8,7 +8,7 @@ spec: - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8080 selector: app.kubernetes.io/name: codeflare-operator app.kubernetes.io/part-of: codeflare From 1ff11205f5f3654a8abb5f0694a812fe1ef18d37 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 24 Jul 2023 13:25:53 +0200 Subject: [PATCH 047/377] Add Ray cluster support in test support package --- test/support/conditions.go | 15 ++++++++++++++- test/support/ray.go | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/test/support/conditions.go b/test/support/conditions.go index e7c5097a..16b26583 100644 --- a/test/support/conditions.go +++ b/test/support/conditions.go @@ -18,6 +18,7 @@ limitations under the License. package support import ( + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" ) @@ -34,7 +35,10 @@ func ConditionStatus[T conditionType](conditionType T) func(any) corev1.Conditio if c := getJobCondition(o.Status.Conditions, batchv1.JobConditionType(conditionType)); c != nil { return c.Status } - + case *appsv1.Deployment: + if c := getDeploymentCondition(o.Status.Conditions, appsv1.DeploymentConditionType(conditionType)); c != nil { + return c.Status + } } return corev1.ConditionUnknown @@ -51,3 +55,12 @@ func getJobCondition(conditions []batchv1.JobCondition, conditionType batchv1.Jo } return nil } + +func getDeploymentCondition(conditions []appsv1.DeploymentCondition, conditionType appsv1.DeploymentConditionType) *appsv1.DeploymentCondition { + for _, c := range conditions { + if c.Type == conditionType { + return &c + } + } + return nil +} diff --git a/test/support/ray.go b/test/support/ray.go index cd5a9d87..52bf2d65 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -68,3 +68,20 @@ func WriteRayJobLogs(t Test, namespace, name string) { t.T().Logf("Retrieving RayJob %s/%s logs", namespace, name) WriteToOutputDir(t, name, Log, GetRayJobLogs(t, namespace, name)) } + +func RayCluster(t Test, namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayCluster { + return func(g gomega.Gomega) *rayv1alpha1.RayCluster { + cluster, err := t.Client().Ray().RayV1alpha1().RayClusters(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return cluster + } +} + +func GetRayCluster(t Test, namespace, name string) *rayv1alpha1.RayCluster { + t.T().Helper() + return RayCluster(t, namespace, name)(t) +} + +func RayClusterState(cluster *rayv1alpha1.RayCluster) rayv1alpha1.ClusterState { + return cluster.Status.State +} From c885e0aeaba81922dfe65a45f648f59a81da50bd Mon Sep 17 00:00:00 2001 From: Wen Zhou Date: Sun, 23 Jul 2023 18:07:50 +0200 Subject: [PATCH 048/377] Update: remove duplicated resource and should have lower in order being depend on others --- controllers/mcad_controller.go | 1 - 1 file changed, 1 deletion(-) diff --git a/controllers/mcad_controller.go b/controllers/mcad_controller.go index f21023cd..01f83060 100644 --- a/controllers/mcad_controller.go +++ b/controllers/mcad_controller.go @@ -224,7 +224,6 @@ func (r *MCADReconciler) SetupWithManager(mgr ctrl.Manager) error { }) return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.MCAD{}). - Owns(&appsv1.Deployment{}). Owns(&corev1.ConfigMap{}). Owns(&corev1.Service{}). Owns(&corev1.ServiceAccount{}). From c9929bd16bb1d98f8b3853e56aa310c4168eb1e3 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 25 Jul 2023 09:21:04 +0200 Subject: [PATCH 049/377] Remove waiting loops for kubernetes resources from Makefile --- .github/actions/kind/action.yml | 2 +- .github/workflows/olm_tests.yaml | 19 ++++++++++++++----- Makefile | 20 -------------------- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/.github/actions/kind/action.yml b/.github/actions/kind/action.yml index 58122e8a..268c48a5 100644 --- a/.github/actions/kind/action.yml +++ b/.github/actions/kind/action.yml @@ -35,7 +35,7 @@ runs: EOF' - name: Setup KinD cluster - uses: helm/kind-action@v1.5.0 + uses: helm/kind-action@v1.8.0 with: cluster_name: cluster version: v0.17.0 diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 9e1061c9..6f88029e 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -72,6 +72,11 @@ jobs: # wait for a while to be sure CRDs are installed sleep 1 kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/olm.yaml + echo Wait for default CatalogSource to start + kubectl wait -n ${{ env.CATALOG_SOURCE_NAMESPACE }} catalogsource/${{ env.CATALOG_SOURCE_NAME }} --for=jsonpath='{.status.connectionState.lastObservedState}'=READY --timeout=180s + env: + CATALOG_SOURCE_NAME: "operatorhubio-catalog" + CATALOG_SOURCE_NAMESPACE: "olm" - name: Create openshift-operator namespace and OperatorGroup run: | @@ -87,15 +92,18 @@ jobs: envsubst < .github/resources-olm-upgrade/subscription.yaml > ${{ env.TEMP_DIR }}/subscription.yaml kubectl create -f ${{ env.TEMP_DIR }}/catalogsource.yaml - make wait-for-catalog-source + + echo Wait for CatalogSource ${{ env.CATALOG_SOURCE_NAME }} to start + kubectl wait -n ${{ env.CATALOG_SOURCE_NAMESPACE }} catalogsource/${{ env.CATALOG_SOURCE_NAME }} --for=jsonpath='{.status.connectionState.lastObservedState}'=READY --timeout=180s kubectl create -f ${{ env.TEMP_DIR }}/subscription.yaml echo Waiting for Subscription to be ready - make wait-for-subscription + kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} subscription/${{ env.SUBSCRIPTION_NAME }} --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s echo Waiting for Deployment to be ready - make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators" + timeout 60 bash -c 'until [[ $(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}') ]]; do sleep 5 && echo "$(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}')"; done' + kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} deployment/codeflare-operator-manager --for=condition=Available=true --timeout=60s env: CATALOG_SOURCE_NAME: "codeflare-olm-test" CATALOG_SOURCE_NAMESPACE: "olm" @@ -138,10 +146,11 @@ jobs: kubectl wait --timeout=120s --for=delete pod/${ORIGINAL_POD_NAME} -n openshift-operators echo Waiting for Subscription to be ready - make wait-for-subscription + kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} subscription/${{ env.SUBSCRIPTION_NAME }} --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s echo Waiting for Deployment to be ready - make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators" + timeout 60 bash -c 'until [[ $(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}') ]]; do sleep 5 && echo "$(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}')"; done' + kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} deployment/codeflare-operator-manager --for=condition=Available=true --timeout=60s echo Checking that correct CSV is available CSV_VERSION=$(kubectl get ClusterServiceVersion/codeflare-operator.${VERSION} -n openshift-operators -o json | jq -r .spec.version) diff --git a/Makefile b/Makefile index f10a8735..4aea9453 100644 --- a/Makefile +++ b/Makefile @@ -440,23 +440,3 @@ test-e2e: defaults manifests generate fmt vet ## Run e2e tests. .PHONY: setup-e2e setup-e2e: ## Set up e2e tests. KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh - -# Wait until CatalogSource is in "READY" state -.PHONY: wait-for-catalog-source -wait-for-catalog-source: - timeout 120 bash -c 'while [[ "$$(kubectl get catalogsource/'$(CATALOG_SOURCE_NAME)' -n '$(CATALOG_SOURCE_NAMESPACE)' -o json | jq -r .status.connectionState.lastObservedState)" != "READY" ]]; do sleep 5 && echo "$$(kubectl get catalogsource/'$(CATALOG_SOURCE_NAME)' -n '$(CATALOG_SOURCE_NAMESPACE)' -o json | jq -r .status.connectionState.lastObservedState)" ; done' - -# Wait until Subscription is in "AtLatestKnown" state -.PHONY: wait-for-subscription -wait-for-subscription: - timeout 300 bash -c 'while [[ "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" != "AtLatestKnown" ]]; do sleep 5 && echo "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" ; done' - -# Default timeout for waiting actions -TIMEOUT ?= 180 - -# Wait until Deployment is in "Available" state -.PHONY: wait-for-deployment -wait-for-deployment: - # Wait until Deployment exists first, then use kubectl wait - timeout $(TIMEOUT) bash -c 'until [[ $$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)') ]]; do sleep 5 && echo "$$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)')"; done' - kubectl wait --timeout=$(TIMEOUT)s --for=condition=Available=true deployment/$(DEPLOYMENT_NAME) -n $(DEPLOYMENT_NAMESPACE) From a82274750112f5d01b18f3de029549a720f12320 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 27 Jul 2023 13:46:17 +0200 Subject: [PATCH 050/377] Add Ray cluster REST API support in test support package --- go.mod | 6 +- go.sum | 10 ++- test/support/client.go | 14 ++++ test/support/conditions.go | 15 ++++ test/support/mcad.go | 3 +- test/support/ray.go | 3 +- test/support/ray_api.go | 45 +++++++++++ test/support/ray_cluster_client.go | 115 +++++++++++++++++++++++++++++ test/support/route.go | 33 +++++++++ 9 files changed, 236 insertions(+), 8 deletions(-) create mode 100644 test/support/ray_api.go create mode 100644 test/support/ray_cluster_client.go create mode 100644 test/support/route.go diff --git a/go.mod b/go.mod index 7d8c01c8..8cbc5ec9 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,8 @@ require ( github.com/manifestival/manifestival v0.7.2 github.com/onsi/ginkgo/v2 v2.9.2 github.com/onsi/gomega v1.27.6 + github.com/openshift/api v0.0.0-20230213134911-7ba313770556 + github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 go.uber.org/zap v1.24.0 @@ -36,7 +38,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.1.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/google/uuid v1.1.2 // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -56,7 +58,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/net v0.8.0 // indirect - golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect + golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/term v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect diff --git a/go.sum b/go.sum index c143ff7a..815e362b 100644 --- a/go.sum +++ b/go.sum @@ -280,8 +280,9 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -418,6 +419,10 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= +github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A= +github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= +github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -635,8 +640,9 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/test/support/client.go b/test/support/client.go index a8efa76d..73931d64 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -24,6 +24,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" + routev1 "github.com/openshift/client-go/route/clientset/versioned" + codeflareclient "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned" rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" @@ -31,6 +33,7 @@ import ( type Client interface { Core() kubernetes.Interface + Route() routev1.Interface CodeFlare() codeflareclient.Interface MCAD() mcadclient.Interface Ray() rayclient.Interface @@ -38,6 +41,7 @@ type Client interface { type testClient struct { core kubernetes.Interface + route routev1.Interface codeflare codeflareclient.Interface mcad mcadclient.Interface ray rayclient.Interface @@ -49,6 +53,10 @@ func (t *testClient) Core() kubernetes.Interface { return t.core } +func (t *testClient) Route() routev1.Interface { + return t.route +} + func (t *testClient) CodeFlare() codeflareclient.Interface { return t.codeflare } @@ -75,6 +83,11 @@ func newTestClient() (Client, error) { return nil, err } + routeClient, err := routev1.NewForConfig(cfg) + if err != nil { + return nil, err + } + codeFlareClient, err := codeflareclient.NewForConfig(cfg) if err != nil { return nil, err @@ -92,6 +105,7 @@ func newTestClient() (Client, error) { return &testClient{ core: kubeClient, + route: routeClient, codeflare: codeFlareClient, mcad: mcadClient, ray: rayClient, diff --git a/test/support/conditions.go b/test/support/conditions.go index 16b26583..70d0ebe9 100644 --- a/test/support/conditions.go +++ b/test/support/conditions.go @@ -21,6 +21,8 @@ import ( appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + + routev1 "github.com/openshift/api/route/v1" ) type conditionType interface { @@ -39,6 +41,10 @@ func ConditionStatus[T conditionType](conditionType T) func(any) corev1.Conditio if c := getDeploymentCondition(o.Status.Conditions, appsv1.DeploymentConditionType(conditionType)); c != nil { return c.Status } + case *routev1.Route: + if c := getRouteCondition(o.Status.Ingress[0].Conditions, routev1.RouteIngressConditionType(conditionType)); c != nil { + return c.Status + } } return corev1.ConditionUnknown @@ -64,3 +70,12 @@ func getDeploymentCondition(conditions []appsv1.DeploymentCondition, conditionTy } return nil } + +func getRouteCondition(conditions []routev1.RouteIngressCondition, conditionType routev1.RouteIngressConditionType) *routev1.RouteIngressCondition { + for _, c := range conditions { + if c.Type == conditionType { + return &c + } + } + return nil +} diff --git a/test/support/mcad.go b/test/support/mcad.go index 4f268985..607a94f7 100644 --- a/test/support/mcad.go +++ b/test/support/mcad.go @@ -18,11 +18,10 @@ package support import ( "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) func AppWrapper(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *mcadv1beta1.AppWrapper { diff --git a/test/support/ray.go b/test/support/ray.go index 52bf2d65..4dc6ebc1 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -20,10 +20,9 @@ import ( "encoding/json" "github.com/onsi/gomega" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" ) const RayJobDefaultClusterSelectorKey = "ray.io/cluster" diff --git a/test/support/ray_api.go b/test/support/ray_api.go new file mode 100644 index 00000000..9004d92e --- /dev/null +++ b/test/support/ray_api.go @@ -0,0 +1,45 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" +) + +func GetRayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) *RayJobDetailsResponse { + t.T().Helper() + return RayJobAPIDetails(t, rayClient, jobID)(t) +} + +func WriteRayJobAPILogs(t Test, rayClient RayClusterClient, jobID string) { + t.T().Helper() + logs, err := rayClient.GetJobLogs(jobID) + t.Expect(err).NotTo(gomega.HaveOccurred()) + WriteToOutputDir(t, jobID, Log, []byte(logs)) +} + +func RayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) func(g gomega.Gomega) *RayJobDetailsResponse { + return func(g gomega.Gomega) *RayJobDetailsResponse { + jobDetails, err := rayClient.GetJobDetails(jobID) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return jobDetails + } +} + +func GetRayJobAPIDetailsStatus(jobDetails *RayJobDetailsResponse) string { + return jobDetails.Status +} diff --git a/test/support/ray_cluster_client.go b/test/support/ray_cluster_client.go new file mode 100644 index 00000000..18dc0c29 --- /dev/null +++ b/test/support/ray_cluster_client.go @@ -0,0 +1,115 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "net/url" +) + +type RayJobSetup struct { + EntryPoint string `json:"entrypoint"` + RuntimeEnv map[string]any `json:"runtime_env"` +} + +type RayJobResponse struct { + JobID string `json:"job_id"` + SubmissionID string `json:"submission_id"` +} + +type RayJobDetailsResponse struct { + JobID string `json:"job_id"` + SubmissionID string `json:"submission_id"` + Status string `json:"status"` +} + +type RayJobLogsResponse struct { + Logs string `json:"logs"` +} + +var _ RayClusterClient = (*rayClusterClient)(nil) + +type rayClusterClient struct { + endpoint url.URL +} + +type RayClusterClient interface { + CreateJob(job *RayJobSetup) (*RayJobResponse, error) + GetJobDetails(jobID string) (*RayJobDetailsResponse, error) + GetJobLogs(jobID string) (string, error) +} + +func NewRayClusterClient(dashboardEndpoint url.URL) RayClusterClient { + return &rayClusterClient{endpoint: dashboardEndpoint} +} + +func (client *rayClusterClient) CreateJob(job *RayJobSetup) (response *RayJobResponse, err error) { + marshalled, err := json.Marshal(job) + if err != nil { + return + } + + createJobURL := client.endpoint.String() + "/api/jobs/" + resp, err := http.Post(createJobURL, "application/json", bytes.NewReader(marshalled)) + if err != nil { + return + } + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = json.Unmarshal(respData, &response) + return +} + +func (client *rayClusterClient) GetJobDetails(jobID string) (response *RayJobDetailsResponse, err error) { + createJobURL := client.endpoint.String() + "/api/jobs/" + jobID + resp, err := http.Get(createJobURL) + if err != nil { + return + } + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = json.Unmarshal(respData, &response) + return +} + +func (client *rayClusterClient) GetJobLogs(jobID string) (logs string, err error) { + createJobURL := client.endpoint.String() + "/api/jobs/" + jobID + "/logs" + resp, err := http.Get(createJobURL) + if err != nil { + return + } + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + jobLogs := RayJobLogsResponse{} + err = json.Unmarshal(respData, &jobLogs) + return jobLogs.Logs, err +} diff --git a/test/support/route.go b/test/support/route.go new file mode 100644 index 00000000..b105643f --- /dev/null +++ b/test/support/route.go @@ -0,0 +1,33 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + routev1 "github.com/openshift/api/route/v1" +) + +func Route(t Test, namespace, name string) func(g gomega.Gomega) *routev1.Route { + return func(g gomega.Gomega) *routev1.Route { + route, err := t.Client().Route().RouteV1().Routes(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return route + } +} From 3f5e55d18428313466cfb9567712f0c94fe8c599 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 27 Jul 2023 13:55:04 -0400 Subject: [PATCH 051/377] Add Make Target for organizing go imports Signed-off-by: Anish Asthana --- Makefile | 14 +++++++ go.mod | 15 ++++++++ go.sum | 85 ++++++++++++++++++++++++++++++++++++++++++ hack/tools.go | 9 +++++ hack/verify-imports.sh | 13 +++++++ 5 files changed, 136 insertions(+) create mode 100644 hack/tools.go create mode 100755 hack/verify-imports.sh diff --git a/Makefile b/Makefile index 4aea9453..bb9b8ac9 100644 --- a/Makefile +++ b/Makefile @@ -278,6 +278,7 @@ LISTER_GEN ?= $(LOCALBIN)/lister-gen INFORMER_GEN ?= $(LOCALBIN)/informer-gen CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest +OPENSHIFT-GOIMPORTS ?= $(LOCALBIN)/openshift-goimports OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk GH_CLI ?= $(LOCALBIN)/gh @@ -338,6 +339,11 @@ envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest +.PHONY: openshift-goimports +openshift-goimports: $(OPENSHIFT-GOIMPORTS) ## Download openshift-goimports locally if necessary. +$(OPENSHIFT-GOIMPORTS): $(LOCALBIN) + test -s $(LOCALBIN)/openshift-goimports || GOBIN=$(LOCALBIN) go install github.com/openshift-eng/openshift-goimports@latest + OPERATOR_SDK_DL_URL := https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION) .PHONY: install-operator-sdk install-operator-sdk: $(OPERATOR_SDK) ## Download fixed version operator-sdk binary for consist outcome @@ -440,3 +446,11 @@ test-e2e: defaults manifests generate fmt vet ## Run e2e tests. .PHONY: setup-e2e setup-e2e: ## Set up e2e tests. KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh + +.PHONY: imports +imports: openshift-goimports ## Organize imports in go files using openshift-goimports. Example: make imports + $(OPENSHIFT-GOIMPORTS) + +.PHONY: verify-imports +verify-imports: openshift-goimports ## Run import verifications. + ./hack/verify-imports.sh $(OPENSHIFT-GOIMPORTS) diff --git a/go.mod b/go.mod index 8cbc5ec9..764c68b3 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/manifestival/manifestival v0.7.2 github.com/onsi/ginkgo/v2 v2.9.2 github.com/onsi/gomega v1.27.6 + github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 @@ -41,22 +42,35 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/google/uuid v1.1.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/magiconair/properties v1.8.4 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml v1.8.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/spf13/afero v1.4.1 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/cobra v1.6.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.7.1 // indirect + github.com/subosito/gotenv v1.2.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect + golang.org/x/mod v0.9.0 // indirect golang.org/x/net v0.8.0 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect golang.org/x/sys v0.6.0 // indirect @@ -68,6 +82,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.26.1 // indirect diff --git a/go.sum b/go.sum index 815e362b..f398a979 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,7 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -68,7 +69,10 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -78,6 +82,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -92,6 +97,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -100,6 +106,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -307,6 +314,8 @@ github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3i github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -317,10 +326,28 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -329,6 +356,8 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -344,6 +373,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -352,6 +383,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -361,6 +393,9 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQSeYtvkY= +github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -375,14 +410,25 @@ github.com/manifestival/manifestival v0.6.0/go.mod h1:3Qq9cnPy7sv7FVhg2Kvw0ebb35 github.com/manifestival/manifestival v0.7.2 h1:l4uFdWX/xQK4QcRfqGoMtBvaZeWPEuwD6hVsCwUqZY4= github.com/manifestival/manifestival v0.7.2/go.mod h1:nl3T6HlfHCeidooWVTMI9vYNTBkQ1GdhLNb+smozbdk= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -419,19 +465,26 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 h1:Zq1BYSO2UmZuu/O1tpYIaC/7ir7ljFqdEY90TwqlseI= +github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2/go.mod h1:VV07Ee+14Mjpwh/ZxtwgAieiCvZ20YVUB9jLHixzswk= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 h1:17lwqvxcWBTi9245lVZgJPvksGwAGA58s8dl7PzpbyE= github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0/go.mod h1:fmbU5LuV1Z2Sbu1FCEoVuw8qxDFcalXvkPyMfGZHHTc= @@ -471,25 +524,42 @@ github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9/g github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ= +github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= +github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -501,6 +571,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -545,12 +617,14 @@ go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -586,12 +660,16 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -654,9 +732,11 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -746,6 +826,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -761,6 +842,7 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -891,6 +973,9 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/hack/tools.go b/hack/tools.go new file mode 100644 index 00000000..f23ba362 --- /dev/null +++ b/hack/tools.go @@ -0,0 +1,9 @@ +//go:build tools +// +build tools + +package hack + +// Add tools that hack scripts depend on here, to ensure they are vendored. +import ( + goimports "github.com/openshift-eng/openshift-goimports" +) diff --git a/hack/verify-imports.sh b/hack/verify-imports.sh new file mode 100755 index 00000000..b823a84b --- /dev/null +++ b/hack/verify-imports.sh @@ -0,0 +1,13 @@ +#!/bin/bash +OPENSHIFT_GOIMPORTS=${1} + +# ${OPENSHIFT_GOIMPORTS} -l +bad_files=$(${OPENSHIFT_GOIMPORTS} -l) + +echo $bad_files +if [[ -n ${bad_files} ]]; then + echo "!!! openshift-goimports needs to be run on the following files:" + echo "${bad_files}" + echo "Try running 'make imports'" + exit 1 +fi From 9b5edf8cbb04c5857aef5608424f72328290cff6 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 27 Jul 2023 17:53:32 -0400 Subject: [PATCH 052/377] Organize go imports Signed-off-by: Anish Asthana --- client/applyconfiguration/utils.go | 3 ++- client/clientset/versioned/clientset.go | 3 ++- client/clientset/versioned/fake/clientset_generated.go | 7 ++++--- client/clientset/versioned/fake/register.go | 3 ++- client/clientset/versioned/scheme/register.go | 3 ++- .../typed/codeflare/v1alpha1/codeflare_client.go | 3 ++- .../codeflare/v1alpha1/fake/fake_codeflare_client.go | 3 ++- .../typed/codeflare/v1alpha1/fake/fake_instascale.go | 5 +++-- .../typed/codeflare/v1alpha1/fake/fake_mcad.go | 5 +++-- .../versioned/typed/codeflare/v1alpha1/instascale.go | 7 ++++--- .../versioned/typed/codeflare/v1alpha1/mcad.go | 7 ++++--- .../externalversions/codeflare/v1alpha1/instascale.go | 9 +++++---- .../externalversions/codeflare/v1alpha1/mcad.go | 9 +++++---- client/informer/externalversions/factory.go | 7 ++++--- client/informer/externalversions/generic.go | 3 ++- .../internalinterfaces/factory_interfaces.go | 3 ++- client/listers/codeflare/v1alpha1/instascale.go | 3 ++- client/listers/codeflare/v1alpha1/mcad.go | 3 ++- controllers/config/manifest.go | 1 + controllers/instascale_controller.go | 7 +++---- controllers/instascale_controller_test.go | 5 ++--- controllers/mcad_controller.go | 1 - controllers/mcad_controller_test.go | 5 ++--- controllers/mcad_params.go | 1 + controllers/suite_test.go | 4 +--- main.go | 8 +++----- test/e2e/mnist_pytorch_mcad_job_test.go | 2 +- test/e2e/mnist_raycluster_sdk_test.go | 5 ++--- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 4 ++-- test/support/client.go | 10 +++++----- 30 files changed, 75 insertions(+), 64 deletions(-) diff --git a/client/applyconfiguration/utils.go b/client/applyconfiguration/utils.go index fd87ac36..44a5e7aa 100644 --- a/client/applyconfiguration/utils.go +++ b/client/applyconfiguration/utils.go @@ -19,9 +19,10 @@ limitations under the License. package applyconfiguration import ( + schema "k8s.io/apimachinery/pkg/runtime/schema" + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" - schema "k8s.io/apimachinery/pkg/runtime/schema" ) // ForKind returns an apply configuration type for the given GroupVersionKind, or nil if no diff --git a/client/clientset/versioned/clientset.go b/client/clientset/versioned/clientset.go index 1d91a664..ca6d68fb 100644 --- a/client/clientset/versioned/clientset.go +++ b/client/clientset/versioned/clientset.go @@ -22,10 +22,11 @@ import ( "fmt" "net/http" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" discovery "k8s.io/client-go/discovery" rest "k8s.io/client-go/rest" flowcontrol "k8s.io/client-go/util/flowcontrol" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" ) type Interface interface { diff --git a/client/clientset/versioned/fake/clientset_generated.go b/client/clientset/versioned/fake/clientset_generated.go index fbbb80dc..b3e9cbb9 100644 --- a/client/clientset/versioned/fake/clientset_generated.go +++ b/client/clientset/versioned/fake/clientset_generated.go @@ -19,14 +19,15 @@ limitations under the License. package fake import ( - clientset "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" - fakecodeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1/fake" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" fakediscovery "k8s.io/client-go/discovery/fake" "k8s.io/client-go/testing" + + clientset "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" + fakecodeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1/fake" ) // NewSimpleClientset returns a clientset that will respond with the provided objects. diff --git a/client/clientset/versioned/fake/register.go b/client/clientset/versioned/fake/register.go index 4ff1cb3a..05244eb5 100644 --- a/client/clientset/versioned/fake/register.go +++ b/client/clientset/versioned/fake/register.go @@ -19,12 +19,13 @@ limitations under the License. package fake import ( - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) var scheme = runtime.NewScheme() diff --git a/client/clientset/versioned/scheme/register.go b/client/clientset/versioned/scheme/register.go index 743173ce..1a52d7da 100644 --- a/client/clientset/versioned/scheme/register.go +++ b/client/clientset/versioned/scheme/register.go @@ -19,12 +19,13 @@ limitations under the License. package scheme import ( - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) var Scheme = runtime.NewScheme() diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go b/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go index 4b93fe6f..bbe38940 100644 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go @@ -21,9 +21,10 @@ package v1alpha1 import ( "net/http" + rest "k8s.io/client-go/rest" + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" - rest "k8s.io/client-go/rest" ) type CodeflareV1alpha1Interface interface { diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go index 9e234aca..0a18ef5d 100644 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go @@ -19,9 +19,10 @@ limitations under the License. package fake import ( - v1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" ) type FakeCodeflareV1alpha1 struct { diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go index 8abbdc79..fd6bced0 100644 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go @@ -23,13 +23,14 @@ import ( json "encoding/json" "fmt" - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" testing "k8s.io/client-go/testing" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" ) // FakeInstaScales implements InstaScaleInterface diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go index 027f2817..a06c229a 100644 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go @@ -23,13 +23,14 @@ import ( json "encoding/json" "fmt" - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" testing "k8s.io/client-go/testing" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" ) // FakeMCADs implements MCADInterface diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go b/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go index c146a98d..73cc95d3 100644 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go @@ -24,13 +24,14 @@ import ( "fmt" "time" - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" - scheme "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" + scheme "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" ) // InstaScalesGetter has a method to return a InstaScaleInterface. diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go b/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go index 7d70e403..23587f12 100644 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go +++ b/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go @@ -24,13 +24,14 @@ import ( "fmt" "time" - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" - scheme "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" + scheme "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" ) // MCADsGetter has a method to return a MCADInterface. diff --git a/client/informer/externalversions/codeflare/v1alpha1/instascale.go b/client/informer/externalversions/codeflare/v1alpha1/instascale.go index 70346e65..79d6c7ac 100644 --- a/client/informer/externalversions/codeflare/v1alpha1/instascale.go +++ b/client/informer/externalversions/codeflare/v1alpha1/instascale.go @@ -22,14 +22,15 @@ import ( "context" time "time" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" - v1alpha1 "github.com/project-codeflare/codeflare-operator/client/listers/codeflare/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" + v1alpha1 "github.com/project-codeflare/codeflare-operator/client/listers/codeflare/v1alpha1" ) // InstaScaleInformer provides access to a shared informer and lister for diff --git a/client/informer/externalversions/codeflare/v1alpha1/mcad.go b/client/informer/externalversions/codeflare/v1alpha1/mcad.go index 2fe4d5a3..4cda692b 100644 --- a/client/informer/externalversions/codeflare/v1alpha1/mcad.go +++ b/client/informer/externalversions/codeflare/v1alpha1/mcad.go @@ -22,14 +22,15 @@ import ( "context" time "time" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" - v1alpha1 "github.com/project-codeflare/codeflare-operator/client/listers/codeflare/v1alpha1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" + + codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" + versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" + v1alpha1 "github.com/project-codeflare/codeflare-operator/client/listers/codeflare/v1alpha1" ) // MCADInformer provides access to a shared informer and lister for diff --git a/client/informer/externalversions/factory.go b/client/informer/externalversions/factory.go index e1f7f14a..0d3733cb 100644 --- a/client/informer/externalversions/factory.go +++ b/client/informer/externalversions/factory.go @@ -23,13 +23,14 @@ import ( sync "sync" time "time" - versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - codeflare "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/codeflare" - internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" + + versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" + codeflare "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/codeflare" + internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" ) // SharedInformerOption defines the functional option type for SharedInformerFactory. diff --git a/client/informer/externalversions/generic.go b/client/informer/externalversions/generic.go index 9eea54eb..6fa936ba 100644 --- a/client/informer/externalversions/generic.go +++ b/client/informer/externalversions/generic.go @@ -21,9 +21,10 @@ package externalversions import ( "fmt" - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) // GenericInformer is type of SharedIndexInformer which will locate and delegate to other diff --git a/client/informer/externalversions/internalinterfaces/factory_interfaces.go b/client/informer/externalversions/internalinterfaces/factory_interfaces.go index b380f668..0e89555b 100644 --- a/client/informer/externalversions/internalinterfaces/factory_interfaces.go +++ b/client/informer/externalversions/internalinterfaces/factory_interfaces.go @@ -21,10 +21,11 @@ package internalinterfaces import ( time "time" - versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" cache "k8s.io/client-go/tools/cache" + + versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" ) // NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. diff --git a/client/listers/codeflare/v1alpha1/instascale.go b/client/listers/codeflare/v1alpha1/instascale.go index ee5c2284..c2519af5 100644 --- a/client/listers/codeflare/v1alpha1/instascale.go +++ b/client/listers/codeflare/v1alpha1/instascale.go @@ -19,10 +19,11 @@ limitations under the License. package v1alpha1 import ( - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) // InstaScaleLister helps list InstaScales. diff --git a/client/listers/codeflare/v1alpha1/mcad.go b/client/listers/codeflare/v1alpha1/mcad.go index 8ac44bfc..e2f47832 100644 --- a/client/listers/codeflare/v1alpha1/mcad.go +++ b/client/listers/codeflare/v1alpha1/mcad.go @@ -19,10 +19,11 @@ limitations under the License. package v1alpha1 import ( - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" + + v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) // MCADLister helps list MCADs. diff --git a/controllers/config/manifest.go b/controllers/config/manifest.go index 2360100d..bfff07c9 100644 --- a/controllers/config/manifest.go +++ b/controllers/config/manifest.go @@ -4,6 +4,7 @@ import ( "github.com/go-logr/logr" mfc "github.com/manifestival/controller-runtime-client" mf "github.com/manifestival/manifestival" + "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/controllers/instascale_controller.go b/controllers/instascale_controller.go index 80e38a53..5fba37e9 100644 --- a/controllers/instascale_controller.go +++ b/controllers/instascale_controller.go @@ -21,6 +21,9 @@ import ( "fmt" "path" + "github.com/go-logr/logr" + mf "github.com/manifestival/manifestival" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" authv1 "k8s.io/api/rbac/v1" @@ -28,7 +31,6 @@ import ( apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -36,9 +38,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - "github.com/go-logr/logr" - mf "github.com/manifestival/manifestival" - "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" "github.com/project-codeflare/codeflare-operator/controllers/config" "github.com/project-codeflare/codeflare-operator/controllers/util" diff --git a/controllers/instascale_controller_test.go b/controllers/instascale_controller_test.go index 4071c515..1e880190 100644 --- a/controllers/instascale_controller_test.go +++ b/controllers/instascale_controller_test.go @@ -3,11 +3,10 @@ package controllers import ( "context" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - mfc "github.com/manifestival/controller-runtime-client" mf "github.com/manifestival/manifestival" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) diff --git a/controllers/mcad_controller.go b/controllers/mcad_controller.go index 01f83060..906bb7bc 100644 --- a/controllers/mcad_controller.go +++ b/controllers/mcad_controller.go @@ -29,7 +29,6 @@ import ( apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" diff --git a/controllers/mcad_controller_test.go b/controllers/mcad_controller_test.go index 35e892c3..7ffd30cc 100644 --- a/controllers/mcad_controller_test.go +++ b/controllers/mcad_controller_test.go @@ -3,11 +3,10 @@ package controllers import ( "context" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - mfc "github.com/manifestival/controller-runtime-client" mf "github.com/manifestival/manifestival" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) diff --git a/controllers/mcad_params.go b/controllers/mcad_params.go index 5992bede..846e4c73 100644 --- a/controllers/mcad_params.go +++ b/controllers/mcad_params.go @@ -18,6 +18,7 @@ package controllers import ( mf "github.com/manifestival/manifestival" + mcadv1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" ) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 07442278..5ee347ba 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -22,10 +22,9 @@ import ( "testing" "time" + mf "github.com/manifestival/manifestival" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - mf "github.com/manifestival/manifestival" "go.uber.org/zap/zapcore" appsv1 "k8s.io/api/apps/v1" @@ -36,7 +35,6 @@ import ( "k8s.io/client-go/kubernetes/scheme" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" diff --git a/main.go b/main.go index 5358e04b..51d4dbc9 100644 --- a/main.go +++ b/main.go @@ -23,20 +23,18 @@ import ( "go.uber.org/zap/zapcore" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - + _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" "github.com/project-codeflare/codeflare-operator/controllers" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. // +kubebuilder:scaffold:imports ) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 168c7b13..88eca341 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -20,6 +20,7 @@ import ( "testing" . "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -27,7 +28,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/project-codeflare/codeflare-operator/test/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) // Trains the MNIST dataset as a batch Job managed by MCAD, and asserts successful completion of the training job. diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index e8bb1f5c..56385b8f 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -20,16 +20,15 @@ import ( "testing" . "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" - . "github.com/project-codeflare/codeflare-operator/test/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) // Creates a Ray cluster, and trains the MNIST dataset using the CodeFlare SDK. diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 958b3047..72645d5b 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -21,14 +21,14 @@ import ( "testing" . "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/project-codeflare/codeflare-operator/test/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" ) // Trains the MNIST dataset as a RayJob, executed by a Ray cluster managed by MCAD, diff --git a/test/support/client.go b/test/support/client.go index 73931d64..c3c4433e 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -17,18 +17,18 @@ limitations under the License. package support import ( - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" + mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned" + rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" "k8s.io/client-go/kubernetes" + _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/tools/clientcmd" routev1 "github.com/openshift/client-go/route/clientset/versioned" codeflareclient "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned" - rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. ) type Client interface { From 234cb41a4db0393e4230faaa8628c7615c47e855 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 27 Jul 2023 18:47:34 -0400 Subject: [PATCH 053/377] Add verify-imports github action Signed-off-by: Anish Asthana --- .github/workflows/verify_imports.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/verify_imports.yml diff --git a/.github/workflows/verify_imports.yml b/.github/workflows/verify_imports.yml new file mode 100644 index 00000000..561b3fd8 --- /dev/null +++ b/.github/workflows/verify_imports.yml @@ -0,0 +1,20 @@ +name: Verify organization of imports +on: + push: + branches: + - '**' + paths: + - '**.go' + tags-ignore: + - 'v*' + pull_request: + paths: + - '**.go' +jobs: + verify-imports: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Verify that imports are organized + run: make verify-imports From 223dffe042b39f3ea4278facb15db6d5131ad5a9 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Fri, 28 Jul 2023 07:32:42 -0400 Subject: [PATCH 054/377] Remove revive check Signed-off-by: Anish Asthana --- .golangci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.golangci.yaml b/.golangci.yaml index e4fc63b6..573c8291 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -9,4 +9,3 @@ linters: - staticcheck - typecheck - unused - - revive From ef43fee9249abfbd837c3acde9c3bf9924639629 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Fri, 28 Jul 2023 08:02:34 -0400 Subject: [PATCH 055/377] Organize go imports for generated files Signed-off-by: Anish Asthana --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bb9b8ac9..a4f47970 100644 --- a/Makefile +++ b/Makefile @@ -173,8 +173,11 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." -.PHONY: generate-client ## Generate client packages -generate-client: code-generator +.PHONY: generate-client ## Generate client packages and organize the goimports +generate-client: generate-client-files imports + +.PHONY: generate-client-files +generate-client-files: code-generator rm -rf client $(APPLYCONFIGURATION_GEN) \ --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ From 817f97e2be1832641dabb5ba81844b270419818a Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 28 Jul 2023 16:08:01 +0200 Subject: [PATCH 056/377] Release workflow - open a PR with changes instead of direct push --- .github/workflows/tag-and-build.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 339d5b6f..a013291f 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -48,9 +48,11 @@ jobs: # Permission required to create a release permissions: contents: write + pull-requests: write env: IMAGE_ORG_BASE: quay.io/${{ github.event.inputs.quay-organization }} + PR_BRANCH_NAME: adjustments-release-${{ github.event.inputs.version }} steps: - uses: actions/checkout@v3 @@ -132,6 +134,25 @@ jobs: with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} file_pattern: 'README.md controllers/defaults.go *.yaml Makefile go.mod go.sum' + create_branch: true + branch: ${{ env.PR_BRANCH_NAME }} + + - name: Create a PR with code changes + run: | + GIT_BRANCH=${GITHUB_REF#refs/heads/} + gh pr create --base "$GIT_BRANCH" --fill --head "${{ env.PR_BRANCH_NAME }}" --label "lgtm" --label "approved" + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + + - name: Wait until PR with code changes is merged + run: | + timeout 3600 bash -c 'until [[ $(gh pr view '${{ env.PR_BRANCH_NAME }}' --json state --jq .state) == "MERGED" ]]; do sleep 5 && echo "$(gh pr view '${{ env.PR_BRANCH_NAME }}' --json state --jq .state)"; done' + env: + GITHUB_TOKEN: ${{ github.TOKEN }} + + - name: Delete remote branch + run: | + git push origin --delete ${{ env.PR_BRANCH_NAME }} - name: Creates a release in GitHub run: | From 9eeacaedf2a23f6889b4fd0e55ff2401e5dc41a9 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 31 Jul 2023 17:39:30 +0200 Subject: [PATCH 057/377] Fix: add missing --output-base option to applyconfiguration-gen --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index a4f47970..c6115a28 100644 --- a/Makefile +++ b/Makefile @@ -183,6 +183,7 @@ generate-client-files: code-generator --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ --go-header-file="hack/boilerplate.go.txt" \ --output-package="github.com/project-codeflare/codeflare-operator/client/applyconfiguration" \ + --output-base="." \ --trim-path-prefix "github.com/project-codeflare/codeflare-operator" $(CLIENT_GEN) \ --input="codeflare/v1alpha1" \ From 47bc1ff1eb91e5ac5d2322b57b3ff263f4236b7c Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 31 Jul 2023 17:40:30 +0200 Subject: [PATCH 058/377] client: Add missing ControllerImage field to apply configuration --- client/applyconfiguration/codeflare/v1alpha1/mcadspec.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go b/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go index c7cb170a..9358586b 100644 --- a/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go +++ b/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go @@ -33,6 +33,7 @@ type MCADSpecApplyConfiguration struct { QuotaRestURL *string `json:"quotaRestURL,omitempty"` PodCreationTimeout *int `json:"podCreationTimeout,omitempty"` ControllerResources *v1.ResourceRequirements `json:"controllerResources,omitempty"` + ControllerImage *string `json:"controllerImage,omitempty"` } // MCADSpecApplyConfiguration constructs an declarative configuration of the MCADSpec type for use with @@ -104,3 +105,11 @@ func (b *MCADSpecApplyConfiguration) WithControllerResources(value v1.ResourceRe b.ControllerResources = &value return b } + +// WithControllerImage sets the ControllerImage field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ControllerImage field is set to the value of the last call. +func (b *MCADSpecApplyConfiguration) WithControllerImage(value string) *MCADSpecApplyConfiguration { + b.ControllerImage = &value + return b +} From ca3baf682b6174e0a6424c1fae6734071237630a Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 27 Jun 2023 11:02:15 +0200 Subject: [PATCH 059/377] Create GitHub action to automate CodeFlare project release --- .../workflows/project-codeflare-release.yml | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 .github/workflows/project-codeflare-release.yml diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml new file mode 100644 index 00000000..f3e1b286 --- /dev/null +++ b/.github/workflows/project-codeflare-release.yml @@ -0,0 +1,126 @@ +# This workflow will build and release all components of the CodeFlare project + +name: Project CodeFlare Release +on: + workflow_dispatch: + inputs: + operator-version: + description: 'CodeFlare operator version to be released (for example: v0.0.0)' + required: true + replaces: + description: 'The previous operator semantic version that this release replaces (for example: v0.0.0)' + required: true + mcad-version: + description: 'Version of multi-cluster-app-dispatcher to be released (for example: v0.0.0)' + required: true + codeflare-sdk-version: + description: 'Version of CodeFlare-SDK to be released (for example: v0.0.0)' + required: true + instascale-version: + description: 'Version of InstaScale to be released (for example: v0.0.0)' + required: true + is-stable: + description: 'Select if the built images should be tagged as stable' + required: true + type: boolean + codeflare-repository-organization: + description: 'GitHub organization/user containing repositories used for release' + required: true + default: 'project-codeflare' + quay-organization: + description: 'Quay organization used to push the built images to' + required: true + default: 'project-codeflare' + community-operators-prod-organization: + description: 'Owner of target community-operators-prod repository used to open a PR against' + required: true + default: 'redhat-openshift-ecosystem' + +jobs: + release-mcad: + runs-on: ubuntu-latest + + steps: + - name: Release MCAD + run: | + gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref ${{ github.ref }} --field tag=${{ github.event.inputs.mcad-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + - name: Wait for MCAD run to finish + run: | + # wait for a while for Run to be started + sleep 5 + run_id=$(gh run list --workflow mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --limit 1 --json databaseId --jq .[].databaseId) + gh run watch ${run_id} --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --interval 10 --exit-status + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + release-instascale: + needs: release-mcad + runs-on: ubuntu-latest + + steps: + - name: Release InstaScale + run: | + gh workflow run instascale-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field tag=${{ github.event.inputs.instascale-version }} --field mcad-version=${{ github.event.inputs.mcad-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + - name: Wait for InstaScale run to finish + run: | + # wait for a while for Run to be started + sleep 5 + run_id=$(gh run list --workflow instascale-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --limit 1 --json databaseId --jq .[].databaseId) + gh run watch ${run_id} --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --interval 10 --exit-status + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + release-codeflare-sdk: + runs-on: ubuntu-latest + + steps: + - name: Release CodeFlare SDK + run: | + semver_version="${{ github.event.inputs.codeflare-sdk-version }}" + plain_version="${semver_version:1}" + gh workflow run release.yaml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-sdk --ref ${{ github.ref }} --field release-version=${plain_version} --field is-latest=${{ github.event.inputs.is-stable }} --field quay-organization=${{ github.event.inputs.quay-organization }} + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + - name: Wait for CodeFlare SDK run to finish + run: | + # wait for a while for Run to be started + sleep 5 + run_id=$(gh run list --workflow release.yaml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-sdk --limit 1 --json databaseId --jq .[].databaseId) + gh run watch ${run_id} --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-sdk --interval 10 --exit-status + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + release-codeflare-operator: + needs: [release-mcad, release-instascale, release-codeflare-sdk] + runs-on: ubuntu-latest + + steps: + - name: Release CodeFlare operator + run: | + gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field mcad-version=${{ github.event.inputs.mcad-version }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field instascale-version=${{ github.event.inputs.instascale-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + + - name: Wait for CodeFlare operator run to finish + run: | + # wait for a while for Run to be started + sleep 5 + run_id=$(gh run list --workflow tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --limit 1 --json databaseId --jq .[].databaseId) + gh run watch ${run_id} --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --interval 10 --exit-status + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash From c2d2f881549247e067897b83564543be68b126d3 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 24 Jul 2023 11:27:17 +0100 Subject: [PATCH 060/377] Remove kube-rbac-proxy and open metrics endpoint --- config/default/kustomization.yaml | 8 ++-- config/default/manager_auth_proxy_patch.yaml | 39 ------------------- .../metrics_service.yaml} | 6 +-- config/manager/controller_manager_config.yaml | 2 +- config/manager/manager.yaml | 8 +++- .../rbac/auth_proxy_client_clusterrole.yaml | 9 ----- config/rbac/auth_proxy_role.yaml | 17 -------- config/rbac/auth_proxy_role_binding.yaml | 12 ------ config/rbac/kustomization.yaml | 7 ---- config/rbac/role.yaml | 12 ++++++ controllers/mcad_controller.go | 2 + 11 files changed, 28 insertions(+), 94 deletions(-) delete mode 100644 config/default/manager_auth_proxy_patch.yaml rename config/{rbac/auth_proxy_service.yaml => default/metrics_service.yaml} (79%) delete mode 100644 config/rbac/auth_proxy_client_clusterrole.yaml delete mode 100644 config/rbac/auth_proxy_role.yaml delete mode 100644 config/rbac/auth_proxy_role_binding.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 1bfe8c54..ad203a94 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -25,11 +25,9 @@ bases: # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. # - ../prometheus -patchesStrategicMerge: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- manager_auth_proxy_patch.yaml +resources: +# Add metrics service +- metrics_service.yaml # Mount the controller config file for loading manager configurations # through a ComponentConfig type diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 31e1a364..00000000 --- a/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=0.0.0.0:8080" - - "--leader-elect" diff --git a/config/rbac/auth_proxy_service.yaml b/config/default/metrics_service.yaml similarity index 79% rename from config/rbac/auth_proxy_service.yaml rename to config/default/metrics_service.yaml index c53c92d1..0f420610 100644 --- a/config/rbac/auth_proxy_service.yaml +++ b/config/default/metrics_service.yaml @@ -5,10 +5,10 @@ metadata: namespace: system spec: ports: - - name: https - port: 8443 + - name: metrics + port: 8080 protocol: TCP - targetPort: 8080 + targetPort: metrics selector: app.kubernetes.io/name: codeflare-operator app.kubernetes.io/part-of: codeflare diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml index ea5cfaa9..7741092e 100644 --- a/config/manager/controller_manager_config.yaml +++ b/config/manager/controller_manager_config.yaml @@ -3,7 +3,7 @@ kind: ControllerManagerConfig health: healthProbeBindAddress: :8081 metrics: - bindAddress: 127.0.0.1:8080 + bindAddress: 0.0.0.0:8080 webhook: port: 9443 leaderElection: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 59b41783..a8467ccf 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -35,7 +35,9 @@ spec: - command: - /manager args: - - --leader-elect + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=0.0.0.0:8080" + - "--leader-elect" image: controller:latest imagePullPolicy: Always name: manager @@ -44,6 +46,10 @@ spec: capabilities: drop: - "ALL" + ports: + - containerPort: 8080 + protocol: TCP + name: metrics livenessProbe: httpGet: path: /healthz diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml deleted file mode 100644 index 51a75db4..00000000 --- a/config/rbac/auth_proxy_client_clusterrole.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metrics-reader -rules: -- nonResourceURLs: - - "/metrics" - verbs: - - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 80e1857c..00000000 --- a/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml deleted file mode 100644 index ec7acc0a..00000000 --- a/config/rbac/auth_proxy_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: proxy-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 8a599efd..3c96e817 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -10,10 +10,3 @@ resources: - edit_role_binding.yaml # We are using this binding as mcad requires this role - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d0c21ba6..9805d333 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -44,6 +44,18 @@ rules: - patch - update - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - codeflare.codeflare.dev resources: diff --git a/controllers/mcad_controller.go b/controllers/mcad_controller.go index 906bb7bc..770df03e 100644 --- a/controllers/mcad_controller.go +++ b/controllers/mcad_controller.go @@ -123,6 +123,8 @@ func (r *MCADReconciler) DeleteResource(params *MCADParams, template string, fns // +kubebuilder:rbac:groups=extensions,resources=replicasets,verbs=get;list;watch // +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch // +kubebuilder:rbac:groups=storage.k8s.io,resources=csidrivers;csinodes;csistoragecapacities,verbs=get;list;watch +// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create func (r *MCADReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("namespace", req.Namespace) From 8bf0aab426ecc68287697505fecd20e17d72d1b6 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Mon, 31 Jul 2023 16:02:02 -0400 Subject: [PATCH 061/377] Create consolidated action for verification of auto-generated / formatted artifacts Signed-off-by: Anish Asthana --- .github/workflows/verify_generated_files.yml | 44 ++++++++++++++++++++ .github/workflows/verify_imports.yml | 20 --------- 2 files changed, 44 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/verify_generated_files.yml delete mode 100644 .github/workflows/verify_imports.yml diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml new file mode 100644 index 00000000..5783d425 --- /dev/null +++ b/.github/workflows/verify_generated_files.yml @@ -0,0 +1,44 @@ +name: Verify Generated Files and Import Organization +on: + push: + branches: + - '**' + paths: + - '**.go' + - '**go.mod' + - '**go.sum' + tags-ignore: + - 'v*' + pull_request: + paths: + - '**.go' + - '**go.mod' + - '**go.sum' +jobs: + verify-generated-functions: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Verify that the DeepCopy, DeepCopyInto, and DeepCopyObject method implementations have been generated + run: make generate && git diff --exit-code + + verify-generate-client: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Verify that the latest client has been generated + run: make generate-client && git diff --exit-code + + verify-imports: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Verify that imports are organized + run: make verify-imports + + verify-manifests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Verify that the latest WebhookConfigurations, ClusterRoles, and CustomResourceDefinitions have been generated + run: make manifests && git diff --exit-code diff --git a/.github/workflows/verify_imports.yml b/.github/workflows/verify_imports.yml deleted file mode 100644 index 561b3fd8..00000000 --- a/.github/workflows/verify_imports.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Verify organization of imports -on: - push: - branches: - - '**' - paths: - - '**.go' - tags-ignore: - - 'v*' - pull_request: - paths: - - '**.go' -jobs: - verify-imports: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Verify that imports are organized - run: make verify-imports From 226a5609f9eef63986d7f8a9b7e5f251e16bce2d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 20:00:29 +0000 Subject: [PATCH 062/377] Bump github.com/onsi/ginkgo/v2 from 2.9.2 to 2.11.0 Bumps [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo) from 2.9.2 to 2.11.0. - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/ginkgo/compare/v2.9.2...v2.11.0) --- updated-dependencies: - dependency-name: github.com/onsi/ginkgo/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 18 +++++++++--------- go.sum | 36 ++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 764c68b3..c144579c 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,13 @@ require ( github.com/go-logr/logr v1.2.4 github.com/manifestival/controller-runtime-client v0.4.0 github.com/manifestival/manifestival v0.7.2 - github.com/onsi/ginkgo/v2 v2.9.2 - github.com/onsi/gomega v1.27.6 + github.com/onsi/ginkgo/v2 v2.11.0 + github.com/onsi/gomega v1.27.8 github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 - github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 + github.com/ray-project/kuberay/ray-operator v0.0.0-20230802222355-153f35c9fd14 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 @@ -70,14 +70,14 @@ require ( github.com/subosito/gotenv v1.2.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/mod v0.9.0 // indirect - golang.org/x/net v0.8.0 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/term v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/tools v0.9.3 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index f398a979..ad4b81b9 100644 --- a/go.sum +++ b/go.sum @@ -455,16 +455,16 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= -github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 h1:Zq1BYSO2UmZuu/O1tpYIaC/7ir7ljFqdEY90TwqlseI= github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2/go.mod h1:VV07Ee+14Mjpwh/ZxtwgAieiCvZ20YVUB9jLHixzswk= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= @@ -519,8 +519,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 h1:qIThU9GGqEay/y78y4Y9e1FVfrdkH5MFnT0zEJ9yh0A= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9/go.mod h1:2auArgwD9dXXJz1oc7SqQ4U/rHdpwnrBwG98kr8OWXA= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230802222355-153f35c9fd14 h1:KnqFh/uiqY7Zdwr9fUaefMa4j0rYSLkk9tuE4lmW4Ec= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230802222355-153f35c9fd14/go.mod h1:hqphTv0O5l6hHf4/OtEn4ie/OHPoVydLograwaK5cHI= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -660,8 +660,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -709,8 +709,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -788,14 +788,14 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -805,8 +805,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -871,8 +871,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 8a9d684b631e69b7c919fdab0b47834ce1f4179f Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 8 Aug 2023 15:02:07 +0200 Subject: [PATCH 063/377] Upgrade MCAD to version 1.33.0 --- Makefile | 3 ++- config/crd/mcad/kustomization.yaml | 2 +- controllers/defaults.go | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- test/e2e/mnist_pytorch_mcad_job_test.go | 2 +- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 2 +- test/support/client.go | 2 +- test/support/mcad.go | 2 +- 9 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index c6115a28..b6be34ce 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ BUNDLE_VERSION ?= $(VERSION:v%=%) INSTASCALE_VERSION ?= v0.0.5 # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.32.0 +MCAD_VERSION ?= v1.33.0 # MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources MCAD_REF ?= release-${MCAD_VERSION} MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher @@ -223,6 +223,7 @@ vet: ## Run go vet against code. .PHONY: modules modules: ## Update Go dependencies. go get $(MCAD_REPO)@$(MCAD_VERSION) + go get github.com/ray-project/kuberay/ray-operator .PHONY: build build: modules defaults generate fmt vet ## Build manager binary. diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index a433f833..8ea96b05 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=release-v0.0.0 # kpt-set: ${MCAD_CRD} +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v0.0.0 # kpt-set: ${MCAD_CRD} diff --git a/controllers/defaults.go b/controllers/defaults.go index 4dc0b777..3a0a746d 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,6 +5,6 @@ package controllers // *********************** const ( - MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.32.0" + MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.33.0" InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.5" ) diff --git a/go.mod b/go.mod index c144579c..2987d8fd 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,8 @@ require ( github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 - github.com/ray-project/kuberay/ray-operator v0.0.0-20230802222355-153f35c9fd14 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.33.0 + github.com/ray-project/kuberay/ray-operator v0.0.0-20230807232553-238cb4e945b6 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index ad4b81b9..4e90f974 100644 --- a/go.sum +++ b/go.sum @@ -486,8 +486,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 h1:17lwqvxcWBTi9245lVZgJPvksGwAGA58s8dl7PzpbyE= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0/go.mod h1:fmbU5LuV1Z2Sbu1FCEoVuw8qxDFcalXvkPyMfGZHHTc= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.33.0 h1:6a+MnxcFSlheC7RIPGg3s/QCt5+7dD8mJKwdpST7i70= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.33.0/go.mod h1:0J0BDSaIN5lvlmgw+32FcMqe8SflXHtHByUbHmPl4w8= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -519,8 +519,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230802222355-153f35c9fd14 h1:KnqFh/uiqY7Zdwr9fUaefMa4j0rYSLkk9tuE4lmW4Ec= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230802222355-153f35c9fd14/go.mod h1:hqphTv0O5l6hHf4/OtEn4ie/OHPoVydLograwaK5cHI= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230807232553-238cb4e945b6 h1:0O9dKI7eCPJIzJD/HfzbQgACq4GTr4j+VpwjdS7Qv0w= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230807232553-238cb4e945b6/go.mod h1:hqphTv0O5l6hHf4/OtEn4ie/OHPoVydLograwaK5cHI= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 88eca341..b2b7fa13 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -137,7 +137,7 @@ func TestMNISTPyTorchMCAD(t *testing.T) { }, } - _, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) + _, err = test.Client().MCAD().McadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 72645d5b..e6aaea8f 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -212,7 +212,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, } - aw, err = test.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Create(aw) + aw, err = test.Client().MCAD().McadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) diff --git a/test/support/client.go b/test/support/client.go index c3c4433e..86a895ee 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -17,7 +17,7 @@ limitations under the License. package support import ( - mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned" + mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/versioned" rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" "k8s.io/client-go/kubernetes" diff --git a/test/support/mcad.go b/test/support/mcad.go index 607a94f7..a4d69b75 100644 --- a/test/support/mcad.go +++ b/test/support/mcad.go @@ -26,7 +26,7 @@ import ( func AppWrapper(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *mcadv1beta1.AppWrapper { return func(g gomega.Gomega) *mcadv1beta1.AppWrapper { - aw, err := t.Client().MCAD().ArbV1().AppWrappers(namespace.Name).Get(name, metav1.GetOptions{}) + aw, err := t.Client().MCAD().McadV1beta1().AppWrappers(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) return aw } From cd8b115be6213bc99c37ffa68f5a947708f6f9f8 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Tue, 8 Aug 2023 14:57:51 +0000 Subject: [PATCH 064/377] Update dependency versions for release v0.1.0 --- Makefile | 2 +- README.md | 8 ++++---- controllers/defaults.go | 2 +- .../instascale_test_results/case_1/deployment.yaml | 2 +- .../instascale_test_results/case_2/deployment.yaml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index b6be34ce..e0c85fb4 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.0.5 +INSTASCALE_VERSION ?= v0.0.6 # MCAD_VERSION defines the default version of the MCAD controller MCAD_VERSION ?= v1.33.0 diff --git a/README.md b/README.md index e5f25443..e434d001 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------| -| CodeFlare Operator | v0.0.6 | -| Multi-Cluster App Dispatcher | v1.32.0 | -| CodeFlare-SDK | v0.5.0 | -| InstaScale | v0.0.5 | +| CodeFlare Operator | v0.1.0 | +| Multi-Cluster App Dispatcher | v1.33.0 | +| CodeFlare-SDK | v0.6.1 | +| InstaScale | v0.0.6 | | KubeRay | v0.5.0 | diff --git a/controllers/defaults.go b/controllers/defaults.go index 3a0a746d..026be9df 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -6,5 +6,5 @@ package controllers const ( MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.33.0" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.5" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.6" ) diff --git a/controllers/testdata/instascale_test_results/case_1/deployment.yaml b/controllers/testdata/instascale_test_results/case_1/deployment.yaml index bb398dbb..13470266 100644 --- a/controllers/testdata/instascale_test_results/case_1/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_1/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.5 + image: quay.io/project-codeflare/instascale-controller:v0.0.6 resources: limits: cpu: '2' diff --git a/controllers/testdata/instascale_test_results/case_2/deployment.yaml b/controllers/testdata/instascale_test_results/case_2/deployment.yaml index ef7b8da1..2916c719 100644 --- a/controllers/testdata/instascale_test_results/case_2/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_2/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.5 + image: quay.io/project-codeflare/instascale-controller:v0.0.6 resources: limits: cpu: '1' From 81e8f817598f0fb139f15f57e095f6433ce31308 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 3 Aug 2023 10:22:22 +0200 Subject: [PATCH 065/377] e2e: Use Ray dashboard API to retrieve job logs --- .github/actions/kind/action.yml | 9 ++ test/e2e/mnist_rayjob_mcad_raycluster_test.go | 103 +++++++++++++++++- test/support/ingress.go | 41 +++++++ test/support/ray.go | 26 +---- test/support/ray_cluster_client.go | 8 +- test/support/route.go | 5 + 6 files changed, 164 insertions(+), 28 deletions(-) create mode 100644 test/support/ingress.go diff --git a/.github/actions/kind/action.yml b/.github/actions/kind/action.yml index 268c48a5..59dcafef 100644 --- a/.github/actions/kind/action.yml +++ b/.github/actions/kind/action.yml @@ -47,3 +47,12 @@ runs: echo "KinD cluster:" kubectl cluster-info kubectl describe nodes + + - name: Install Ingress controller + shell: bash + run: | + VERSION=controller-v1.6.4 + echo "Deploying Ingress controller into KinD cluster" + curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/"${VERSION}"/deploy/static/provider/kind/deploy.yaml | sed "s/--publish-status-address=localhost/--report-node-internal-ip-address\\n - --status-update-interval=10/g" | kubectl apply -f - + kubectl annotate ingressclass nginx "ingressclass.kubernetes.io/is-default-class=true" + kubectl -n ingress-nginx wait --timeout=300s --for=condition=Available deployments --all diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index e6aaea8f..037a1229 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -18,6 +18,7 @@ package e2e import ( "encoding/base64" + "net/url" "testing" . "github.com/onsi/gomega" @@ -25,8 +26,12 @@ import ( rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + routev1 "github.com/openshift/api/route/v1" . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -252,8 +257,104 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) + var rayDashboardURL url.URL + if IsOpenShift(test) { + // Create a route to expose the Ray cluster API + route := &routev1.Route{ + TypeMeta: metav1.TypeMeta{ + APIVersion: routev1.GroupVersion.String(), + Kind: "Route", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace.Name, + Name: "ray-dashboard", + }, + Spec: routev1.RouteSpec{ + To: routev1.RouteTargetReference{ + Name: "raycluster-head-svc", + }, + Port: &routev1.RoutePort{ + TargetPort: intstr.FromString("dashboard"), + }, + }, + } + + _, err := test.Client().Route().RouteV1().Routes(namespace.Name).Create(test.Ctx(), route, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Route %s/%s successfully", route.Namespace, route.Name) + + test.T().Logf("Waiting for Route %s/%s to be admitted", route.Namespace, route.Name) + test.Eventually(Route(test, route.Namespace, route.Name), TestTimeoutMedium). + Should(WithTransform(ConditionStatus(routev1.RouteAdmitted), Equal(corev1.ConditionTrue))) + + route = GetRoute(test, route.Namespace, route.Name) + + rayDashboardURL = url.URL{ + Scheme: "http", + Host: route.Status.Ingress[0].Host, + } + } else { + ingress := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: networkingv1.SchemeGroupVersion.String(), + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace.Name, + Name: "ray-dashboard", + Annotations: map[string]string{ + "nginx.ingress.kubernetes.io/use-regex": "true", + "nginx.ingress.kubernetes.io/rewrite-target": "/$2", + }, + }, + Spec: networkingv1.IngressSpec{ + Rules: []networkingv1.IngressRule{ + { + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/ray-dashboard(/|$)(.*)", + PathType: Ptr(networkingv1.PathTypePrefix), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "raycluster-head-svc", + Port: networkingv1.ServiceBackendPort{ + Name: "dashboard", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + _, err := test.Client().Core().NetworkingV1().Ingresses(ingress.Namespace).Create(test.Ctx(), ingress, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Ingress %s/%s successfully", ingress.Namespace, ingress.Name) + + test.T().Logf("Waiting for Ingress %s/%s to be admitted", ingress.Namespace, ingress.Name) + test.Eventually(Ingress(test, ingress.Namespace, ingress.Name), TestTimeoutMedium). + Should(WithTransform(LoadBalancerIngresses, HaveLen(1))) + + ingress = GetIngress(test, ingress.Namespace, ingress.Name) + + rayDashboardURL = url.URL{ + Scheme: "http", + Host: ingress.Status.LoadBalancer.Ingress[0].IP, + Path: "ray-dashboard", + } + } + + test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) + rayClient := NewRayClusterClient(rayDashboardURL) + // Retrieving the job logs once it has completed or timed out - defer WriteRayJobLogs(test, rayJob.Namespace, rayJob.Name) + defer WriteRayJobAPILogs(test, rayClient, GetRayJobId(test, rayJob.Namespace, rayJob.Name)) test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). diff --git a/test/support/ingress.go b/test/support/ingress.go new file mode 100644 index 00000000..667abe6e --- /dev/null +++ b/test/support/ingress.go @@ -0,0 +1,41 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" + + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Ingress(t Test, namespace, name string) func(g gomega.Gomega) *networkingv1.Ingress { + return func(g gomega.Gomega) *networkingv1.Ingress { + ingress, err := t.Client().Core().NetworkingV1().Ingresses(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return ingress + } +} + +func GetIngress(t Test, namespace, name string) *networkingv1.Ingress { + t.T().Helper() + return Ingress(t, namespace, name)(t) +} + +func LoadBalancerIngresses(ingress *networkingv1.Ingress) []networkingv1.IngressLoadBalancerIngress { + return ingress.Status.LoadBalancer.Ingress +} diff --git a/test/support/ray.go b/test/support/ray.go index 4dc6ebc1..fec4ba67 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -17,8 +17,6 @@ limitations under the License. package support import ( - "encoding/json" - "github.com/onsi/gomega" rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" @@ -44,28 +42,10 @@ func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus { return job.Status.JobStatus } -func GetRayJobLogs(t Test, namespace, name string) []byte { +func GetRayJobId(t Test, namespace, name string) string { t.T().Helper() - - job := GetRayJob(t, namespace, name) - - response := t.Client().Core().CoreV1().RESTClient(). - Get(). - AbsPath("/api/v1/namespaces", job.Namespace, "services", "http:"+job.Status.RayClusterName+"-head-svc:dashboard", "proxy", "api", "jobs", job.Status.JobId, "logs"). - Do(t.Ctx()) - t.Expect(response.Error()).NotTo(gomega.HaveOccurred()) - - body := map[string]string{} - bytes, _ := response.Raw() - t.Expect(json.Unmarshal(bytes, &body)).To(gomega.Succeed()) - t.Expect(body).To(gomega.HaveKey("logs")) - - return []byte(body["logs"]) -} - -func WriteRayJobLogs(t Test, namespace, name string) { - t.T().Logf("Retrieving RayJob %s/%s logs", namespace, name) - WriteToOutputDir(t, name, Log, GetRayJobLogs(t, namespace, name)) + job := RayJob(t, namespace, name)(t) + return job.Status.JobId } func RayCluster(t Test, namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayCluster { diff --git a/test/support/ray_cluster_client.go b/test/support/ray_cluster_client.go index 18dc0c29..ce36459b 100644 --- a/test/support/ray_cluster_client.go +++ b/test/support/ray_cluster_client.go @@ -19,7 +19,7 @@ package support import ( "bytes" "encoding/json" - "io/ioutil" + "io" "net/http" "net/url" ) @@ -72,7 +72,7 @@ func (client *rayClusterClient) CreateJob(job *RayJobSetup) (response *RayJobRes return } - respData, err := ioutil.ReadAll(resp.Body) + respData, err := io.ReadAll(resp.Body) if err != nil { return } @@ -88,7 +88,7 @@ func (client *rayClusterClient) GetJobDetails(jobID string) (response *RayJobDet return } - respData, err := ioutil.ReadAll(resp.Body) + respData, err := io.ReadAll(resp.Body) if err != nil { return } @@ -104,7 +104,7 @@ func (client *rayClusterClient) GetJobLogs(jobID string) (logs string, err error return } - respData, err := ioutil.ReadAll(resp.Body) + respData, err := io.ReadAll(resp.Body) if err != nil { return } diff --git a/test/support/route.go b/test/support/route.go index b105643f..8a74ca12 100644 --- a/test/support/route.go +++ b/test/support/route.go @@ -31,3 +31,8 @@ func Route(t Test, namespace, name string) func(g gomega.Gomega) *routev1.Route return route } } + +func GetRoute(t Test, namespace, name string) *routev1.Route { + t.T().Helper() + return Route(t, namespace, name)(t) +} From bc4b46a6818a04ff7243bff8903fa08b771d53aa Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 3 Aug 2023 11:16:36 +0200 Subject: [PATCH 066/377] e2e: Add kind-e2e target to setup test KinD cluster --- Makefile | 4 ++++ README.md | 6 +++++- test/e2e/kind.sh | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100755 test/e2e/kind.sh diff --git a/Makefile b/Makefile index e0c85fb4..77b32679 100644 --- a/Makefile +++ b/Makefile @@ -448,6 +448,10 @@ test-unit: defaults manifests generate fmt vet envtest ## Run unit tests. test-e2e: defaults manifests generate fmt vet ## Run e2e tests. go test -timeout 30m -v ./test/e2e +.PHONY: kind-e2e +setup-e2e: ## Set up e2e KinD cluster. + test/e2e/kind.sh + .PHONY: setup-e2e setup-e2e: ## Set up e2e tests. KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh diff --git a/README.md b/README.md index e434d001..5cdb6e1c 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,15 @@ The e2e tests can be executed locally by running the following commands: ```bash # Create a KinD cluster - $ kind create cluster --image kindest/node:v1.25.8 + $ make kind-e2e # Install the CRDs $ make install ``` + [!NOTE] + Some e2e tests cover the access to services via Ingresses, as end-users would do, which requires access to the Ingress controller load balancer by its IP. + For it to work on macOS, this requires installing [docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect). + 2. Start the operator locally: ```bash diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh new file mode 100755 index 00000000..5b39a164 --- /dev/null +++ b/test/e2e/kind.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright 2022 IBM, Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail +: "${INGRESS_NGINX_VERSION:=controller-v1.6.4}" + +echo "Creating KinD cluster" +cat < Date: Wed, 9 Aug 2023 10:13:11 +0100 Subject: [PATCH 067/377] Small Make target bug-fix --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 77b32679..a3ff4efd 100644 --- a/Makefile +++ b/Makefile @@ -449,7 +449,7 @@ test-e2e: defaults manifests generate fmt vet ## Run e2e tests. go test -timeout 30m -v ./test/e2e .PHONY: kind-e2e -setup-e2e: ## Set up e2e KinD cluster. +kind-e2e: ## Set up e2e KinD cluster. test/e2e/kind.sh .PHONY: setup-e2e From b325fab34bd5569a1f161ba818c90e001f509cd0 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 9 Aug 2023 09:43:00 +0200 Subject: [PATCH 068/377] Adjust SDK version, fix TestMNISTRayClusterSDK --- Makefile | 2 +- test/e2e/mnist_raycluster_sdk.py | 5 ++--- test/e2e/mnist_raycluster_sdk_test.go | 3 +++ test/support/defaults.go | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index a3ff4efd..aa5118ec 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ KUBERAY_VERSION ?= v0.5.0 RAY_VERSION ?= 2.5.0 # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK -CODEFLARE_SDK_VERSION ?= 0.4.4 +CODEFLARE_SDK_VERSION ?= 0.6.1 # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to diff --git a/test/e2e/mnist_raycluster_sdk.py b/test/e2e/mnist_raycluster_sdk.py index 444b1858..b830a004 100644 --- a/test/e2e/mnist_raycluster_sdk.py +++ b/test/e2e/mnist_raycluster_sdk.py @@ -12,13 +12,12 @@ cluster = Cluster(ClusterConfiguration( name='mnist', namespace=namespace, - min_worker=1, - max_worker=1, + num_workers=1, min_cpus='500m', max_cpus=1, min_memory=0.5, max_memory=1, - gpu=0, + num_gpus=0, instascale=False, )) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 56385b8f..28fe0c1d 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -45,6 +45,9 @@ func TestMNISTRayClusterSDK(t *testing.T) { test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/146") } + // Currently blocked by https://github.com/project-codeflare/codeflare-sdk/pull/271 , remove the skip once SDK with the PR is released + test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/271") + // Create a namespace namespace := test.NewTestNamespace() diff --git a/test/support/defaults.go b/test/support/defaults.go index a0aaad42..c87b5b1a 100644 --- a/test/support/defaults.go +++ b/test/support/defaults.go @@ -5,7 +5,7 @@ package support // *********************** const ( - CodeFlareSDKVersion = "0.4.4" + CodeFlareSDKVersion = "0.6.1" RayVersion = "2.5.0" RayImage = "rayproject/ray:2.5.0" ) From 850b8505d71825c8fece9348e5aa626fdc4de7c7 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 9 Aug 2023 15:50:27 +0200 Subject: [PATCH 069/377] Add note to apply SecurityContextConstraints for OpenShift e2e tests --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 5cdb6e1c..c01e3c48 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,26 @@ The e2e tests can be executed locally by running the following commands: $ make setup-e2e ``` + [!NOTE] + In OpenShift the KubeRay operator pod gets random user assigned. This user is then used to run Ray cluster. + However the random user assigned by OpenShift doesn't have rights to store dataset downloaded as part of test execution, causing tests to fail. + To prevent this failure on OpenShift user should enforce user 1000 for KubeRay and Ray cluster by creating this SCC in KubeRay operator namespace (replace the namespace placeholder): + + ```yaml + kind: SecurityContextConstraints + apiVersion: security.openshift.io/v1 + metadata: + name: run-as-ray-user + seLinuxContext: + type: MustRunAs + runAsUser: + type: MustRunAs + uid: 1000 + users: + - 'system:serviceaccount:$(namespace):kuberay-operator' + ``` + + 4. In a separate terminal, run the e2e suite: ```bash From fee5b5c6538c36494fa8e6766e4cc2acb72166cb Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 9 Aug 2023 12:24:36 +0200 Subject: [PATCH 070/377] Upgrade Go to version 1.19 --- .github/workflows/e2e_tests.yaml | 2 +- .github/workflows/olm_tests.yaml | 2 +- CONTRIBUTING.md | 2 +- Dockerfile | 2 +- go.mod | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 04b810fd..318cbb1c 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -57,7 +57,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.18 + go-version: v1.19 - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 6f88029e..6e478df9 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -56,7 +56,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.18 + go-version: v1.19 - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f321c10d..79cae2f0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Here are a few things to go over before getting started with CodeFlare Operator ## Environment setup The following should be installed in your working environment: - - Go 1.18.X + - Go 1.19.x - [Download release](https://go.dev/dl/) - [Install Instructions](https://go.dev/doc/install) - [Operator SDK](https://sdk.operatorframework.io/docs/installation/) diff --git a/Dockerfile b/Dockerfile index b9717367..302b31e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.access.redhat.com/ubi8/go-toolset:1.18.9-8 as builder +FROM registry.access.redhat.com/ubi8/go-toolset:1.19.10-10 as builder WORKDIR /workspace # Copy the Go Modules manifests diff --git a/go.mod b/go.mod index 2987d8fd..74552dd9 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/project-codeflare/codeflare-operator -go 1.18 +go 1.19 require ( github.com/go-logr/logr v1.2.4 From bdd595fce5faebf8ec39f42c10f81eeda2606f05 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 9 Aug 2023 12:38:57 +0200 Subject: [PATCH 071/377] Setup Go 1.19 in pre-commit workflow --- .github/workflows/precommit.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 494e0dd3..7ef3a5e4 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -23,6 +23,11 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.19 + - name: Activate cache uses: actions/cache@v3 with: From 7d8046e7d7e97959a9b9a9328a058a76822645b8 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 10 Aug 2023 16:20:29 +0200 Subject: [PATCH 072/377] Use Go 1.19 everywhere --- .github/workflows/operator-image.yml | 5 +++++ .github/workflows/tag-and-build.yml | 5 +++++ .github/workflows/unit_tests.yml | 5 +++++ .github/workflows/verify_generated_files.yml | 16 ++++++++++++++++ 4 files changed, 31 insertions(+) diff --git a/.github/workflows/operator-image.yml b/.github/workflows/operator-image.yml index 54c6403a..901f894b 100644 --- a/.github/workflows/operator-image.yml +++ b/.github/workflows/operator-image.yml @@ -16,6 +16,11 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.19 + - name: Install operator-sdk run: | export ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index a013291f..2f49cfdb 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -57,6 +57,11 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.19 + - name: Verify that release doesn't exist yet shell: bash {0} run: | diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index f28c5b2b..7615ff77 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -24,6 +24,11 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.19 + - name: Activate cache uses: actions/cache@v3 with: diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml index 5783d425..a5cc6ad2 100644 --- a/.github/workflows/verify_generated_files.yml +++ b/.github/workflows/verify_generated_files.yml @@ -19,6 +19,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.19 - name: Verify that the DeepCopy, DeepCopyInto, and DeepCopyObject method implementations have been generated run: make generate && git diff --exit-code @@ -26,6 +30,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.19 - name: Verify that the latest client has been generated run: make generate-client && git diff --exit-code @@ -33,6 +41,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.19 - name: Verify that imports are organized run: make verify-imports @@ -40,5 +52,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.19 - name: Verify that the latest WebhookConfigurations, ClusterRoles, and CustomResourceDefinitions have been generated run: make manifests && git diff --exit-code From 099be364f98789b36238d3600bed09a3732578e8 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 10 Aug 2023 16:22:08 +0200 Subject: [PATCH 073/377] Add sutaakar to OWNERS --- OWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/OWNERS b/OWNERS index 58754041..efdcada1 100644 --- a/OWNERS +++ b/OWNERS @@ -4,6 +4,7 @@ approvers: - jbusche - kpostoffice - maxusmusti + - sutaakar - tedhtchang reviewers: - anishasthana @@ -13,4 +14,5 @@ reviewers: - kpostoffice - maxusmusti - MichaelClifford + - sutaakar - tedhtchang From 31eddbc5ddebd4b3f00c62ed63cd25f171fb7013 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Wed, 9 Aug 2023 16:13:48 +0100 Subject: [PATCH 074/377] docs: remove dollar signs to allow for correct copy and paste --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c01e3c48..f656e97c 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ The e2e tests can be executed locally by running the following commands: ```bash # Create a KinD cluster - $ make kind-e2e + make kind-e2e # Install the CRDs - $ make install + make install ``` [!NOTE] @@ -37,7 +37,7 @@ The e2e tests can be executed locally by running the following commands: 2. Start the operator locally: ```bash - $ make run + make run ``` Alternatively, You can run the operator from your IDE / debugger. @@ -45,7 +45,7 @@ The e2e tests can be executed locally by running the following commands: 3. Set up the test CodeFlare stack: ```bash - $ make setup-e2e + make setup-e2e ``` [!NOTE] @@ -71,7 +71,7 @@ The e2e tests can be executed locally by running the following commands: 4. In a separate terminal, run the e2e suite: ```bash - $ make test-e2e + make test-e2e ``` Alternatively, You can run the e2e test(s) from your IDE / debugger. From 30a6812ba51e46172338bb18f4d371980fe546fb Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 9 Aug 2023 19:39:24 +0200 Subject: [PATCH 075/377] Adjust CODEFLARE_SDK_VERSION in GitHub release workflow --- .github/workflows/tag-and-build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 2f49cfdb..87925554 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -90,9 +90,12 @@ jobs: sed -i -E "s/(.*CodeFlare-SDK.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.codeflare-sdk-version }}\2/" README.md sed -i -E "s/(.*InstaScale.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" README.md - - name: Adjust MCAD and InstaScale dependencies in the code + - name: Adjust MCAD, SDK and InstaScale dependencies in the code run: | + # Remove leading 'v' + CODEFLARE_SDK_VERSION=$(cut -c2- <<< ${{ github.event.inputs.codeflare-sdk-version }}) sed -i -E "s/(.*MCAD_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" Makefile + sed -i -E "s/(.*CODEFLARE_SDK_VERSION \?= )[0-9]+\.[0-9]+\.[0-9]+(.*)/\1$CODEFLARE_SDK_VERSION\2/" Makefile sed -i -E "s/(.*INSTASCALE_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" Makefile sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_1/deployment.yaml sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_2/deployment.yaml From 2a4731a237cf1996e6370def8c3dc7842fc2db52 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 10 Aug 2023 12:58:39 +0200 Subject: [PATCH 076/377] Build and publish dev operator image for every commit --- .github/workflows/operator-image.yml | 41 +++++++++------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/.github/workflows/operator-image.yml b/.github/workflows/operator-image.yml index 901f894b..d2df8683 100644 --- a/.github/workflows/operator-image.yml +++ b/.github/workflows/operator-image.yml @@ -1,14 +1,17 @@ -# This workflow will build the CodeFlare Operator image and push it to the project-codeflare image registry +# This workflow will build the CodeFlare Operator dev image and push it to the project-codeflare image registry -name: Operator Image +name: Operator Dev Image on: - workflow_dispatch: - inputs: - tag: - description: 'Tag to be used for operator image' - required: true - default: 'unstable' + push: + branches: + - main + paths-ignore: + - 'docs/**' + - 'test/**' + - '**.adoc' + - '**.md' + - 'LICENSE' jobs: push: @@ -21,14 +24,6 @@ jobs: with: go-version: v1.19 - - name: Install operator-sdk - run: | - export ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac) - export OS=$(uname | awk '{print tolower($0)}') - export OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/v1.27.0 - curl -LO ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} - chmod +x operator-sdk_${OS}_${ARCH} && sudo mv operator-sdk_${OS}_${ARCH} /usr/local/bin/operator-sdk - - name: Login to Quay.io uses: redhat-actions/podman-login@v1 with: @@ -39,15 +34,5 @@ jobs: - name: Image Build run: | make build - make bundle - make image-build -e IMG=quay.io/project-codeflare/codeflare-operator:${SOURCE_TAG} - podman tag quay.io/project-codeflare/codeflare-operator:${SOURCE_TAG} quay.io/project-codeflare/codeflare-operator:latest - env: - SOURCE_TAG: ${{ github.event.inputs.tag }} - - - name: Image Push - run: | - make image-push -e IMG=quay.io/project-codeflare/codeflare-operator:${SOURCE_TAG} - make image-push -e IMG=quay.io/project-codeflare/codeflare-operator:latest - env: - SOURCE_TAG: ${{ github.event.inputs.tag }} + make image-build -e IMG=quay.io/project-codeflare/codeflare-operator:dev + make image-push -e IMG=quay.io/project-codeflare/codeflare-operator:dev From 8d4c41367423df441332c9436059c127cf6801f9 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 10 Aug 2023 19:38:38 -0400 Subject: [PATCH 077/377] Update CodeFlare stack release instructions Signed-off-by: Anish Asthana --- README.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f656e97c..a00dbf47 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,6 @@ The e2e tests can be executed locally by running the following commands: - 'system:serviceaccount:$(namespace):kuberay-operator' ``` - 4. In a separate terminal, run the e2e suite: ```bash @@ -78,22 +77,23 @@ The e2e tests can be executed locally by running the following commands: ## Release -Prerequisite: -- Build and release [MCAD](https://github.com/project-codeflare/multi-cluster-app-dispatcher) -- Build and release [InstaScale](https://github.com/project-codeflare/instascale) -- Build and release [CodeFlare-SDK](https://github.com/project-codeflare/codeflare-sdk) - -Release steps: -1. Invoke [tag-and-build.yml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/tag-and-build.yml) GitHub action, this action will create a repository tag, build and push operator image. - -2. Check result of [tag-and-build.yml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/tag-and-build.yml) GitHub action, it should pass. - -3. Verify that compatibility matrix in [README](https://github.com/project-codeflare/codeflare-operator/blob/main/README.md) was properly updated. +1. Invoke [project-codeflare-release.yaml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/project-codeflare-release.yml) +2. Once all jobs within the action are completed, verify that compatibility matrix in [README](https://github.com/project-codeflare/codeflare-operator/blob/main/README.md) was properly updated. +3. Verify that opened pull request to [OpenShift community operators repository](https://github.com/redhat-openshift-ecosystem/community-operators-prod) has proper content. +4. Once PR is merged, announce the new release in slack and mail lists, if any. +5. Update the Distributed Workloads component in ODH (also copy/update the compatibility matrix). This may require yaml and test updates depending on the release. Make sure to create a tag + release in the Distributed Workloads repository that matches the project-codeflare release version. +6. Update the readme/markdown/yaml in odh-manifests as required. -4. Verify that opened pull request to [OpenShift community operators repository](https://github.com/redhat-openshift-ecosystem/community-operators-prod) has proper content. +### Releases involving part of the stack -5. Once PR is merged, update component stable tags to point at the latest image release. +There may be instances in which a new CodeFlare stack release requires releases of only a subset of the stack components. Examples could be hotfixes for a specific component. In these instances: -6. Announce the new release in slack and mail lists, if any. +1. Build updated components as needed: + - Build and release [MCAD](https://github.com/project-codeflare/multi-cluster-app-dispatcher) + - Build and release [InstaScale](https://github.com/project-codeflare/instascale) + - Build and release [CodeFlare-SDK](https://github.com/project-codeflare/codeflare-sdk) -7. Update the Distributed Workloads component in ODH (also copy/update the compatibility matrix). +2. Invoke [tag-and-build.yml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/tag-and-build.yml) GitHub action, this action will create a repository tag, build and push operator image. +3. Check result of [tag-and-build.yml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/tag-and-build.yml) GitHub action, it should pass. +4. Verify that compatibility matrix in [README](https://github.com/project-codeflare/codeflare-operator/blob/main/README.md) was properly updated. +5. Follow the steps 3-6 from the previous section. From 6fdc33354e193b11fee9a6b178983ba47c5a35e3 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Mon, 14 Aug 2023 12:05:44 -0400 Subject: [PATCH 078/377] Update Job names so there is no overlap between e2e and olm CI Signed-off-by: Anish Asthana --- .github/workflows/e2e_tests.yaml | 2 +- .github/workflows/olm_tests.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 318cbb1c..236883ea 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -25,7 +25,7 @@ concurrency: cancel-in-progress: true jobs: - kubernetes: + kubernetes-e2e: runs-on: ubuntu-20.04 diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 6e478df9..67c797be 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -18,7 +18,7 @@ concurrency: cancel-in-progress: true jobs: - kubernetes: + kubernetes-olm-upgrade: runs-on: ubuntu-20.04 timeout-minutes: 60 env: From 66da5abbaa520dae940abce1afe3248a580fb2ea Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 14 Aug 2023 16:45:49 +0100 Subject: [PATCH 079/377] removing skip --- test/e2e/mnist_raycluster_sdk_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 28fe0c1d..1b59eac0 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -41,10 +41,6 @@ func TestMNISTRayClusterSDK(t *testing.T) { test := With(t) test.T().Parallel() - if !IsOpenShift(test) { - test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/146") - } - // Currently blocked by https://github.com/project-codeflare/codeflare-sdk/pull/271 , remove the skip once SDK with the PR is released test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/271") From 39225d3d16fb74f603ee6fb2c7d3b1d91b6b2047 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 15 Aug 2023 08:39:59 +0200 Subject: [PATCH 080/377] Apply MCAD CRD version replace for operator uninstall and undeploy --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index aa5118ec..b3ca3523 100644 --- a/Makefile +++ b/Makefile @@ -255,7 +255,9 @@ install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~ .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + git restore config/* .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. @@ -266,7 +268,9 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + git restore config/* ##@ Build Dependencies From be0dd48b43b415e8fe06bdf0c940d53c088619b3 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 17 Aug 2023 14:14:04 +0200 Subject: [PATCH 081/377] Test support: return default status if Route is not initialized --- test/support/conditions.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/support/conditions.go b/test/support/conditions.go index 70d0ebe9..4ef4be1e 100644 --- a/test/support/conditions.go +++ b/test/support/conditions.go @@ -42,6 +42,10 @@ func ConditionStatus[T conditionType](conditionType T) func(any) corev1.Conditio return c.Status } case *routev1.Route: + if len(o.Status.Ingress) == 0 { + // Route is not initialized yet + break + } if c := getRouteCondition(o.Status.Ingress[0].Conditions, routev1.RouteIngressConditionType(conditionType)); c != nil { return c.Status } From a3db2b2793e8a95da6b582ed77cc3bc02ed032f5 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 22 Aug 2023 15:19:21 +0200 Subject: [PATCH 082/377] Test support: return error if Ray API response code doesn't match --- test/support/ray_cluster_client.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/test/support/ray_cluster_client.go b/test/support/ray_cluster_client.go index ce36459b..bbce9af2 100644 --- a/test/support/ray_cluster_client.go +++ b/test/support/ray_cluster_client.go @@ -19,6 +19,7 @@ package support import ( "bytes" "encoding/json" + "fmt" "io" "net/http" "net/url" @@ -77,13 +78,17 @@ func (client *rayClusterClient) CreateJob(job *RayJobSetup) (response *RayJobRes return } + if resp.StatusCode != 200 { + return nil, fmt.Errorf("incorrect response code: %d for creating Ray Job, response body: %s", resp.StatusCode, respData) + } + err = json.Unmarshal(respData, &response) return } func (client *rayClusterClient) GetJobDetails(jobID string) (response *RayJobDetailsResponse, err error) { - createJobURL := client.endpoint.String() + "/api/jobs/" + jobID - resp, err := http.Get(createJobURL) + getJobDetailsURL := client.endpoint.String() + "/api/jobs/" + jobID + resp, err := http.Get(getJobDetailsURL) if err != nil { return } @@ -93,13 +98,17 @@ func (client *rayClusterClient) GetJobDetails(jobID string) (response *RayJobDet return } + if resp.StatusCode != 200 { + return nil, fmt.Errorf("incorrect response code: %d for retrieving Ray Job details, response body: %s", resp.StatusCode, respData) + } + err = json.Unmarshal(respData, &response) return } func (client *rayClusterClient) GetJobLogs(jobID string) (logs string, err error) { - createJobURL := client.endpoint.String() + "/api/jobs/" + jobID + "/logs" - resp, err := http.Get(createJobURL) + getJobLogsURL := client.endpoint.String() + "/api/jobs/" + jobID + "/logs" + resp, err := http.Get(getJobLogsURL) if err != nil { return } @@ -109,6 +118,10 @@ func (client *rayClusterClient) GetJobLogs(jobID string) (logs string, err error return } + if resp.StatusCode != 200 { + return "", fmt.Errorf("incorrect response code: %d for retrieving Ray Job logs, response body: %s", resp.StatusCode, respData) + } + jobLogs := RayJobLogsResponse{} err = json.Unmarshal(respData, &jobLogs) return jobLogs.Logs, err From 6f7d6b1194dd88aa0b2ab08b5b0e1856793883dd Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 23 Aug 2023 10:49:04 +0200 Subject: [PATCH 083/377] Test support: Return WriteRayJobLogs function --- test/support/ray.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/support/ray.go b/test/support/ray.go index fec4ba67..b69609ae 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -64,3 +64,7 @@ func GetRayCluster(t Test, namespace, name string) *rayv1alpha1.RayCluster { func RayClusterState(cluster *rayv1alpha1.RayCluster) rayv1alpha1.ClusterState { return cluster.Status.State } + +func WriteRayJobLogs(t Test, rayClient RayClusterClient, namespace, name string) { + WriteRayJobAPILogs(t, rayClient, GetRayJobId(t, namespace, name)) +} From 5209f7a7cea17b303127bd5132ec9e02a476ab84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Aug 2023 13:45:44 +0000 Subject: [PATCH 084/377] build(deps): bump github.com/onsi/gomega from 1.27.8 to 1.27.10 Bumps [github.com/onsi/gomega](https://github.com/onsi/gomega) from 1.27.8 to 1.27.10. - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/gomega/compare/v1.27.8...v1.27.10) --- updated-dependencies: - dependency-name: github.com/onsi/gomega dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 74552dd9..31067991 100644 --- a/go.mod +++ b/go.mod @@ -7,12 +7,12 @@ require ( github.com/manifestival/controller-runtime-client v0.4.0 github.com/manifestival/manifestival v0.7.2 github.com/onsi/ginkgo/v2 v2.11.0 - github.com/onsi/gomega v1.27.8 + github.com/onsi/gomega v1.27.10 github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/multi-cluster-app-dispatcher v1.33.0 - github.com/ray-project/kuberay/ray-operator v0.0.0-20230807232553-238cb4e945b6 + github.com/ray-project/kuberay/ray-operator v0.0.0-20230810043804-80a6d58e2e5d go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index 4e90f974..80d12494 100644 --- a/go.sum +++ b/go.sum @@ -463,8 +463,8 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= -github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 h1:Zq1BYSO2UmZuu/O1tpYIaC/7ir7ljFqdEY90TwqlseI= github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2/go.mod h1:VV07Ee+14Mjpwh/ZxtwgAieiCvZ20YVUB9jLHixzswk= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= @@ -519,8 +519,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230807232553-238cb4e945b6 h1:0O9dKI7eCPJIzJD/HfzbQgACq4GTr4j+VpwjdS7Qv0w= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230807232553-238cb4e945b6/go.mod h1:hqphTv0O5l6hHf4/OtEn4ie/OHPoVydLograwaK5cHI= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230810043804-80a6d58e2e5d h1:FpE2VQLDFWRymSP6vmdG+xY0fbtk3VcT/D53WWOPZsc= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230810043804-80a6d58e2e5d/go.mod h1:hqphTv0O5l6hHf4/OtEn4ie/OHPoVydLograwaK5cHI= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= From d36bb3cee1b018bad4c0d5756208861013590476 Mon Sep 17 00:00:00 2001 From: anishasthana Date: Tue, 29 Aug 2023 16:36:18 +0000 Subject: [PATCH 085/377] Update dependency versions for release v0.2.0 --- Makefile | 6 +- README.md | 8 +- controllers/defaults.go | 4 +- .../case_1/deployment.yaml | 2 +- .../case_2/deployment.yaml | 2 +- go.mod | 26 +++--- go.sum | 90 +++++++++++++------ 7 files changed, 85 insertions(+), 53 deletions(-) diff --git a/Makefile b/Makefile index b3ca3523..52cd6181 100644 --- a/Makefile +++ b/Makefile @@ -12,10 +12,10 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.0.6 +INSTASCALE_VERSION ?= v.0.0.7 # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.33.0 +MCAD_VERSION ?= v1.34.0 # MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources MCAD_REF ?= release-${MCAD_VERSION} MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher @@ -30,7 +30,7 @@ KUBERAY_VERSION ?= v0.5.0 RAY_VERSION ?= 2.5.0 # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK -CODEFLARE_SDK_VERSION ?= 0.6.1 +CODEFLARE_SDK_VERSION ?= 0.7.0 # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to diff --git a/README.md b/README.md index a00dbf47..72d9ee02 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------| -| CodeFlare Operator | v0.1.0 | -| Multi-Cluster App Dispatcher | v1.33.0 | -| CodeFlare-SDK | v0.6.1 | -| InstaScale | v0.0.6 | +| CodeFlare Operator | v0.2.0 | +| Multi-Cluster App Dispatcher | v1.34.0 | +| CodeFlare-SDK | v0.7.0 | +| InstaScale | v.0.0.7 | | KubeRay | v0.5.0 | diff --git a/controllers/defaults.go b/controllers/defaults.go index 026be9df..dc7fc960 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,6 +5,6 @@ package controllers // *********************** const ( - MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.33.0" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.6" + MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.34.0" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v.0.0.7" ) diff --git a/controllers/testdata/instascale_test_results/case_1/deployment.yaml b/controllers/testdata/instascale_test_results/case_1/deployment.yaml index 13470266..686d7a0e 100644 --- a/controllers/testdata/instascale_test_results/case_1/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_1/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.6 + image: quay.io/project-codeflare/instascale-controller:v.0.0.7 resources: limits: cpu: '2' diff --git a/controllers/testdata/instascale_test_results/case_2/deployment.yaml b/controllers/testdata/instascale_test_results/case_2/deployment.yaml index 2916c719..a86df3a5 100644 --- a/controllers/testdata/instascale_test_results/case_2/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_2/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.6 + image: quay.io/project-codeflare/instascale-controller:v.0.0.7 resources: limits: cpu: '1' diff --git a/go.mod b/go.mod index 31067991..fd1ec1cb 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,8 @@ require ( github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/multi-cluster-app-dispatcher v1.33.0 - github.com/ray-project/kuberay/ray-operator v0.0.0-20230810043804-80a6d58e2e5d + github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0 + github.com/ray-project/kuberay/ray-operator v0.0.0-20230829051452-55b1d39efb62 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 @@ -25,22 +25,22 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/zapr v1.2.3 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.19.14 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/swag v0.22.3 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.1.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -48,7 +48,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.4 // indirect - github.com/mailru/easyjson v0.7.6 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect @@ -87,9 +87,9 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.26.1 // indirect k8s.io/component-base v0.26.2 // indirect - k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect - k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d // indirect + k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 80d12494..cbede5ca 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -84,6 +85,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -94,6 +96,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -125,11 +129,13 @@ github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkg github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= +github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= @@ -143,6 +149,7 @@ github.com/evanphx/json-patch/v5 v5.2.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2Vvl github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -188,15 +195,15 @@ github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwds github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= @@ -219,8 +226,8 @@ github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/ github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= @@ -271,8 +278,8 @@ github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Z github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= +github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -299,8 +306,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -326,6 +333,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -387,6 +395,8 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -402,8 +412,8 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/manifestival/controller-runtime-client v0.4.0 h1:AK+nTZ1rhPTfAc3upnSe3TiOvAWuRmxi1vjEr55dSUM= github.com/manifestival/controller-runtime-client v0.4.0/go.mod h1:DrsviSegfdlpVsynTUZonKE5gFkMQupenlzVmDbC6S0= github.com/manifestival/manifestival v0.6.0/go.mod h1:3Qq9cnPy7sv7FVhg2Kvw0ebb35R4OdctS4UjTjHlHm4= @@ -444,7 +454,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -486,8 +495,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.33.0 h1:6a+MnxcFSlheC7RIPGg3s/QCt5+7dD8mJKwdpST7i70= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.33.0/go.mod h1:0J0BDSaIN5lvlmgw+32FcMqe8SflXHtHByUbHmPl4w8= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0 h1:4J3vW+vuCBHyocd3huZy3FJhOw5acaBHFLYwutkTE/E= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0/go.mod h1:gtTl8Tsl+X+bGhqVudLoveINR6IkN+sVvH0J+VZIP40= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -519,9 +528,10 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230810043804-80a6d58e2e5d h1:FpE2VQLDFWRymSP6vmdG+xY0fbtk3VcT/D53WWOPZsc= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230810043804-80a6d58e2e5d/go.mod h1:hqphTv0O5l6hHf4/OtEn4ie/OHPoVydLograwaK5cHI= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230829051452-55b1d39efb62 h1:rmdUxgzlhwWdZ4D0fBmdW8Y9kgkQ+Kv8kqr/BvFKKqo= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230829051452-55b1d39efb62/go.mod h1:6d2QFMJ0OY2KzIUH3wtdwsvgRiimc1/8ioQLZr1ljKk= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -564,13 +574,18 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -579,6 +594,9 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -597,6 +615,7 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -705,7 +724,9 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -781,7 +802,9 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -803,6 +826,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= @@ -927,6 +951,7 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -934,6 +959,7 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -947,6 +973,9 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -959,14 +988,16 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -983,6 +1014,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1037,20 +1069,20 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kube-openapi v0.0.0-20200204173128-addea2498afe/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d h1:VcFq5n7wCJB2FQMCIHfC+f+jNcGgNMar1uKd6rVlifU= +k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= +k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= @@ -1058,8 +1090,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQb sigs.k8s.io/controller-runtime v0.7.2/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= From 65c0cad1078eb89829da21baafea14fc06f38463 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Fri, 25 Aug 2023 18:09:26 -0400 Subject: [PATCH 086/377] Automatically label issues as needing triage and add them to project board Signed-off-by: Anish Asthana --- .github/workflows/auto-add-issues.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/auto-add-issues.yml diff --git a/.github/workflows/auto-add-issues.yml b/.github/workflows/auto-add-issues.yml new file mode 100644 index 00000000..a8be31eb --- /dev/null +++ b/.github/workflows/auto-add-issues.yml @@ -0,0 +1,25 @@ +name: Label new issues as needs-triage and add to CodeFlare Sprint Board +on: + issues: + types: + - opened +jobs: + add_label: + name: Add needs-triage label to new issues + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - uses: actions/checkout@v3 + - run: gh issue edit ${{ github.event.issue.number }} --add-label "triage/needs-triage" + env: + GH_TOKEN: ${{ github.token }} + + add-to-project: + name: Add issue to project + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@v0.5.0 + with: + project-url: https://github.com/orgs/project-codeflare/projects/8 + github-token: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} From 150ca3a7fa2efb63c8b22cc55fdc0e2020182e3b Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Wed, 30 Aug 2023 12:39:30 +0100 Subject: [PATCH 087/377] Fixing instascale version --- Makefile | 2 +- README.md | 2 +- controllers/defaults.go | 2 +- .../testdata/instascale_test_results/case_1/deployment.yaml | 2 +- .../testdata/instascale_test_results/case_2/deployment.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 52cd6181..5a33e4b0 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v.0.0.7 +INSTASCALE_VERSION ?= v0.0.7 # MCAD_VERSION defines the default version of the MCAD controller MCAD_VERSION ?= v1.34.0 diff --git a/README.md b/README.md index 72d9ee02..b798a3ee 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ CodeFlare Stack Compatibility Matrix | CodeFlare Operator | v0.2.0 | | Multi-Cluster App Dispatcher | v1.34.0 | | CodeFlare-SDK | v0.7.0 | -| InstaScale | v.0.0.7 | +| InstaScale | v0.0.7 | | KubeRay | v0.5.0 | diff --git a/controllers/defaults.go b/controllers/defaults.go index dc7fc960..dfcb4a28 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -6,5 +6,5 @@ package controllers const ( MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.34.0" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v.0.0.7" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.7" ) diff --git a/controllers/testdata/instascale_test_results/case_1/deployment.yaml b/controllers/testdata/instascale_test_results/case_1/deployment.yaml index 686d7a0e..4e3c3078 100644 --- a/controllers/testdata/instascale_test_results/case_1/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_1/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v.0.0.7 + image: quay.io/project-codeflare/instascale-controller:v0.0.7 resources: limits: cpu: '2' diff --git a/controllers/testdata/instascale_test_results/case_2/deployment.yaml b/controllers/testdata/instascale_test_results/case_2/deployment.yaml index a86df3a5..bb8eeeea 100644 --- a/controllers/testdata/instascale_test_results/case_2/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_2/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v.0.0.7 + image: quay.io/project-codeflare/instascale-controller:v0.0.7 resources: limits: cpu: '1' From e309be97f2f8973ed851c46cefdc858cd27941c4 Mon Sep 17 00:00:00 2001 From: anishasthana Date: Wed, 30 Aug 2023 12:23:57 +0000 Subject: [PATCH 088/377] Update dependency versions for release v0.2.2 --- README.md | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b798a3ee..fe5ce178 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------| -| CodeFlare Operator | v0.2.0 | +| CodeFlare Operator | v0.2.2 | | Multi-Cluster App Dispatcher | v1.34.0 | | CodeFlare-SDK | v0.7.0 | | InstaScale | v0.0.7 | diff --git a/go.mod b/go.mod index fd1ec1cb..764faf2e 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0 - github.com/ray-project/kuberay/ray-operator v0.0.0-20230829051452-55b1d39efb62 + github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index cbede5ca..b87ec39e 100644 --- a/go.sum +++ b/go.sum @@ -528,8 +528,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230829051452-55b1d39efb62 h1:rmdUxgzlhwWdZ4D0fBmdW8Y9kgkQ+Kv8kqr/BvFKKqo= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230829051452-55b1d39efb62/go.mod h1:6d2QFMJ0OY2KzIUH3wtdwsvgRiimc1/8ioQLZr1ljKk= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576 h1:NYt+ipGo+jMLYCFI5S04DzVFSXn8WCR0m6m3bFXPk+4= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576/go.mod h1:7PBzsR4L0b/f5tTxq1XGnzvudbrMyPadiLiNiP9pOl0= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= From 72e9e64fb64721cbb2a451089758dab57afc5fae Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 31 Aug 2023 08:48:53 +0200 Subject: [PATCH 089/377] e2e tests: Reduce CPU requests for e2e test AppWrappers --- test/e2e/mnist_pytorch_mcad_job_test.go | 2 +- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index b2b7fa13..82f5b808 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -121,7 +121,7 @@ func TestMNISTPyTorchMCAD(t *testing.T) { { Replicas: 1, Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceCPU: resource.MustParse("100m"), corev1.ResourceMemory: resource.MustParse("512Mi"), }, Limits: corev1.ResourceList{ diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 037a1229..bf618cbf 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -201,7 +201,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { { Replicas: 2, Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceCPU: resource.MustParse("100m"), corev1.ResourceMemory: resource.MustParse("512Mi"), }, Limits: corev1.ResourceList{ From 88221fa195b7ebaf7baff30748d34252a2cbfac9 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Wed, 30 Aug 2023 15:40:53 -0400 Subject: [PATCH 090/377] Change remaining is-latest references to stable Signed-off-by: Anish Asthana --- .github/workflows/project-codeflare-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index f3e1b286..629ac81d 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -88,7 +88,7 @@ jobs: run: | semver_version="${{ github.event.inputs.codeflare-sdk-version }}" plain_version="${semver_version:1}" - gh workflow run release.yaml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-sdk --ref ${{ github.ref }} --field release-version=${plain_version} --field is-latest=${{ github.event.inputs.is-stable }} --field quay-organization=${{ github.event.inputs.quay-organization }} + gh workflow run release.yaml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-sdk --ref ${{ github.ref }} --field release-version=${plain_version} --field is-stable=${{ github.event.inputs.is-stable }} --field quay-organization=${{ github.event.inputs.quay-organization }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash From 486c24114b1b8a451c2c9780cb128b73faf573e0 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 1 Sep 2023 12:35:30 +0200 Subject: [PATCH 091/377] Release workflow - replace any version value in Makefile --- .github/workflows/tag-and-build.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 87925554..26667caa 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -94,11 +94,12 @@ jobs: run: | # Remove leading 'v' CODEFLARE_SDK_VERSION=$(cut -c2- <<< ${{ github.event.inputs.codeflare-sdk-version }}) - sed -i -E "s/(.*MCAD_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" Makefile - sed -i -E "s/(.*CODEFLARE_SDK_VERSION \?= )[0-9]+\.[0-9]+\.[0-9]+(.*)/\1$CODEFLARE_SDK_VERSION\2/" Makefile - sed -i -E "s/(.*INSTASCALE_VERSION \?= )v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" Makefile - sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_1/deployment.yaml - sed -i -E "s/(.*instascale-controller:)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" controllers/testdata/instascale_test_results/case_2/deployment.yaml + sed -i -E "s/(.*MCAD_VERSION \?= ).*/\1${{ github.event.inputs.mcad-version }}/" Makefile + sed -i -E "s/(.*MCAD_REF \?= ).*/\1release-\${MCAD_VERSION}/" Makefile + sed -i -E "s/(.*CODEFLARE_SDK_VERSION \?= ).*/\1$CODEFLARE_SDK_VERSION/" Makefile + sed -i -E "s/(.*INSTASCALE_VERSION \?= ).*/\1${{ github.event.inputs.instascale-version }}/" Makefile + sed -i -E "s/(.*instascale-controller:).*/\1${{ github.event.inputs.instascale-version }}/" controllers/testdata/instascale_test_results/case_1/deployment.yaml + sed -i -E "s/(.*instascale-controller:).*/\1${{ github.event.inputs.instascale-version }}/" controllers/testdata/instascale_test_results/case_2/deployment.yaml - name: Login to Quay.io uses: redhat-actions/podman-login@v1 @@ -141,7 +142,7 @@ jobs: uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} - file_pattern: 'README.md controllers/defaults.go *.yaml Makefile go.mod go.sum' + file_pattern: 'README.md */defaults.go *.yaml Makefile go.mod go.sum' create_branch: true branch: ${{ env.PR_BRANCH_NAME }} From bf0fd98040850d51196911808002671805a54284 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 4 Sep 2023 13:19:07 +0100 Subject: [PATCH 092/377] Remove custom-metrics-api --- Makefile | 2 +- .../mcad/apiservice_custom-metrics.yaml | 18 ------------------ ...rrole_custom-metrics-server-admin.yaml.tmpl | 13 ------------- ...usterrole_metrics-resource-reader.yaml.tmpl | 18 ------------------ ...ing_hpa-controller-custom-metrics.yaml.tmpl | 16 ---------------- ...inding_mcad-system-auth-delegator.yaml.tmpl | 16 ---------------- ...lebinding_metrics-resource-reader.yaml.tmpl | 16 ---------------- ...inding_custom-metrics-auth-reader.yaml.tmpl | 13 ------------- controllers/multi_cluster_app_dispatcher.go | 6 ------ 9 files changed, 1 insertion(+), 117 deletions(-) delete mode 100644 config/internal/mcad/apiservice_custom-metrics.yaml delete mode 100644 config/internal/mcad/clusterrole_custom-metrics-server-admin.yaml.tmpl delete mode 100644 config/internal/mcad/clusterrole_metrics-resource-reader.yaml.tmpl delete mode 100644 config/internal/mcad/clusterrolebinding_hpa-controller-custom-metrics.yaml.tmpl delete mode 100644 config/internal/mcad/clusterrolebinding_mcad-system-auth-delegator.yaml.tmpl delete mode 100644 config/internal/mcad/clusterrolebinding_metrics-resource-reader.yaml.tmpl delete mode 100644 config/internal/mcad/rolebinding_custom-metrics-auth-reader.yaml.tmpl diff --git a/Makefile b/Makefile index 5a33e4b0..77b172c3 100644 --- a/Makefile +++ b/Makefile @@ -453,7 +453,7 @@ test-e2e: defaults manifests generate fmt vet ## Run e2e tests. go test -timeout 30m -v ./test/e2e .PHONY: kind-e2e -kind-e2e: ## Set up e2e KinD cluster. +kind-e2e: ## Set up e2e KinD cluster test/e2e/kind.sh .PHONY: setup-e2e diff --git a/config/internal/mcad/apiservice_custom-metrics.yaml b/config/internal/mcad/apiservice_custom-metrics.yaml deleted file mode 100644 index b054fee1..00000000 --- a/config/internal/mcad/apiservice_custom-metrics.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# {{ if (eq .Values.configMap.multiCluster true) }} -apiVersion: apiregistration.k8s.io/v1beta1 -kind: APIService -metadata: - name: v1beta1.custom.metrics.k8s.io - labels: - app.kubernetes.io/managed-by: MCAD - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} -spec: - service: - name: mcad-{{.Name}}-metrics - namespace: {{.Namespace}} - group: custom.metrics.k8s.io - version: v1beta1 - insecureSkipTLSVerify: true - groupPriorityMinimum: 100 - versionPriority: 100 diff --git a/config/internal/mcad/clusterrole_custom-metrics-server-admin.yaml.tmpl b/config/internal/mcad/clusterrole_custom-metrics-server-admin.yaml.tmpl deleted file mode 100644 index e7500795..00000000 --- a/config/internal/mcad/clusterrole_custom-metrics-server-admin.yaml.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{.Name}}-mcad-custom-metrics-server-admin-role - labels: - app.kubernetes.io/managed-by: MCAD - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} -rules: - - apiGroups: - - custom.metrics.k8s.io - resources: ["*"] - verbs: ["*"] diff --git a/config/internal/mcad/clusterrole_metrics-resource-reader.yaml.tmpl b/config/internal/mcad/clusterrole_metrics-resource-reader.yaml.tmpl deleted file mode 100644 index 99f12b69..00000000 --- a/config/internal/mcad/clusterrole_metrics-resource-reader.yaml.tmpl +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{.Name}}-mcad-metrics-resource-reader-role - labels: - app.kubernetes.io/managed-by: MCAD - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} -rules: - - apiGroups: - - "" - resources: - - namespaces - - pods - - services - verbs: - - get - - list diff --git a/config/internal/mcad/clusterrolebinding_hpa-controller-custom-metrics.yaml.tmpl b/config/internal/mcad/clusterrolebinding_hpa-controller-custom-metrics.yaml.tmpl deleted file mode 100644 index e4a482fe..00000000 --- a/config/internal/mcad/clusterrolebinding_hpa-controller-custom-metrics.yaml.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: hpa-controller-custom-metrics -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{.Name}}-mcad-custom-metrics-server-admin-role - labels: - app.kubernetes.io/managed-by: MCAD - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} -subjects: - - kind: ServiceAccount - name: horizontal-pod-autoscaler - namespace: kube-system diff --git a/config/internal/mcad/clusterrolebinding_mcad-system-auth-delegator.yaml.tmpl b/config/internal/mcad/clusterrolebinding_mcad-system-auth-delegator.yaml.tmpl deleted file mode 100644 index fc66e588..00000000 --- a/config/internal/mcad/clusterrolebinding_mcad-system-auth-delegator.yaml.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{.Name}}-mcad-system:auth-delegator - labels: - app.kubernetes.io/managed-by: MCAD - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: - - kind: ServiceAccount - name: mcad-controller-{{.Name}} - namespace: {{.Namespace}} diff --git a/config/internal/mcad/clusterrolebinding_metrics-resource-reader.yaml.tmpl b/config/internal/mcad/clusterrolebinding_metrics-resource-reader.yaml.tmpl deleted file mode 100644 index 9d4327e4..00000000 --- a/config/internal/mcad/clusterrolebinding_metrics-resource-reader.yaml.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{.Name}}-mcad-metrics-resource-reader-crb - labels: - app.kubernetes.io/managed-by: MCAD - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{.Name}}-mcad-metrics-resource-reader-role -subjects: - - kind: ServiceAccount - name: mcad-controller-{{.Name}} - namespace: {{.Namespace}} diff --git a/config/internal/mcad/rolebinding_custom-metrics-auth-reader.yaml.tmpl b/config/internal/mcad/rolebinding_custom-metrics-auth-reader.yaml.tmpl deleted file mode 100644 index f8794545..00000000 --- a/config/internal/mcad/rolebinding_custom-metrics-auth-reader.yaml.tmpl +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{.Name}}-custom-metrics-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: - - kind: ServiceAccount - name: mcad-controller-{{.Name}} - namespace: {{.Namespace}} diff --git a/controllers/multi_cluster_app_dispatcher.go b/controllers/multi_cluster_app_dispatcher.go index f49d9049..ca3b7e79 100644 --- a/controllers/multi_cluster_app_dispatcher.go +++ b/controllers/multi_cluster_app_dispatcher.go @@ -27,16 +27,10 @@ var multiClusterAppDispatcherTemplates = []string{ "mcad/deployment.yaml.tmpl", } var ownerLessmultiClusterAppDispatcherTemplates = []string{ - "mcad/rolebinding_custom-metrics-auth-reader.yaml.tmpl", - "mcad/clusterrole_custom-metrics-server-admin.yaml.tmpl", "mcad/clusterrole_mcad-controller.yaml.tmpl", - "mcad/clusterrole_metrics-resource-reader.yaml.tmpl", - "mcad/clusterrolebinding_hpa-controller-custom-metrics.yaml.tmpl", "mcad/clusterrolebinding_mcad-controller.yaml.tmpl", "mcad/clusterrolebinding_mcad-controller-kube-scheduler.yaml.tmpl", "mcad/clusterrolebinding_mcad-edit.yaml.tmpl", - "mcad/clusterrolebinding_mcad-system-auth-delegator.yaml.tmpl", - "mcad/clusterrolebinding_metrics-resource-reader.yaml.tmpl", } func (r *MCADReconciler) ReconcileMCAD(mcad *codeflarev1alpha1.MCAD, params *MCADParams) error { From c6b02440cdf65ea80ac14137dca77892df9efa5b Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 25 Aug 2023 14:44:29 +0200 Subject: [PATCH 093/377] Move AppWrapper API to workload.codeflare.dev group --- .yamllint.yaml | 2 +- Makefile | 6 +-- .../internal/instascale/clusterrole.yaml.tmpl | 2 +- .../clusterrole_mcad-controller.yaml.tmpl | 2 +- config/rbac/role.yaml | 48 +++++++------------ controllers/defaults.go | 4 +- controllers/instascale_controller.go | 1 - controllers/mcad_controller.go | 2 +- .../case_1/clusterrole.yaml | 2 +- go.mod | 4 +- go.sum | 4 +- test/e2e/mnist_pytorch_mcad_job_test.go | 2 +- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 2 +- test/support/mcad.go | 2 +- 14 files changed, 35 insertions(+), 48 deletions(-) diff --git a/.yamllint.yaml b/.yamllint.yaml index 75b22773..768bc626 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -1,5 +1,5 @@ ignore: | - bundle/manifests/mcad.ibm.com* + bundle/manifests/* extends: default rules: line-length: disable diff --git a/Makefile b/Makefile index 77b172c3..6d498b2c 100644 --- a/Makefile +++ b/Makefile @@ -12,12 +12,12 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.0.7 +INSTASCALE_VERSION ?= dev # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.34.0 +MCAD_VERSION ?= main # MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources -MCAD_REF ?= release-${MCAD_VERSION} +MCAD_REF ?= dev MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) # The image is still published using the MCAD_REF format (i.e release-vX.Y.Z) diff --git a/config/internal/instascale/clusterrole.yaml.tmpl b/config/internal/instascale/clusterrole.yaml.tmpl index 8f27f57f..90e60d37 100644 --- a/config/internal/instascale/clusterrole.yaml.tmpl +++ b/config/internal/instascale/clusterrole.yaml.tmpl @@ -60,7 +60,7 @@ rules: - patch - apiGroups: - - mcad.ibm.com + - workload.codeflare.dev resources: - appwrappers verbs: diff --git a/config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl b/config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl index 7d49f5fe..49dfe097 100644 --- a/config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl +++ b/config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl @@ -11,7 +11,7 @@ metadata: codeflare.codeflare.dev/cr-namespace: {{.Namespace}} rules: - apiGroups: - - mcad.ibm.com + - workload.codeflare.dev resources: - queuejobs - schedulingspecs diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 9805d333..691c9dfb 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -262,37 +262,6 @@ rules: - patch - update - watch -- apiGroups: - - mcad.ibm.com - resources: - - appwrappers - - appwrappers/finalizers - - appwrappers/status - - queuejobs - - schedulingspecs - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch -- apiGroups: - - mcad.ibm.com - resources: - - appwrappers - - queuejobs - - schedulingspecs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - policy resources: @@ -349,3 +318,20 @@ rules: - get - list - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers + - appwrappers/finalizers + - appwrappers/status + - queuejobs + - schedulingspecs + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch diff --git a/controllers/defaults.go b/controllers/defaults.go index dfcb4a28..bac326fd 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,6 +5,6 @@ package controllers // *********************** const ( - MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.34.0" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.7" + MCADImage = "quay.io/project-codeflare/mcad-controller:dev" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:dev" ) diff --git a/controllers/instascale_controller.go b/controllers/instascale_controller.go index 5fba37e9..9fca52a0 100644 --- a/controllers/instascale_controller.go +++ b/controllers/instascale_controller.go @@ -88,7 +88,6 @@ func (r *InstaScaleReconciler) Apply(owner mf.Owner, params *InstaScaleParams, t // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings,verbs=get;list;watch;create;update;delete // +kubebuilder:rbac:groups=machine.openshift.io,resources=*,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=mcad.ibm.com,resources=appwrappers;queuejobs;schedulingspecs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=config.openshift.io,resources=clusterversions,verbs=get;list // Reconcile is part of the main kubernetes reconciliation loop which aims to diff --git a/controllers/mcad_controller.go b/controllers/mcad_controller.go index 770df03e..9dfecc24 100644 --- a/controllers/mcad_controller.go +++ b/controllers/mcad_controller.go @@ -102,7 +102,7 @@ func (r *MCADReconciler) DeleteResource(params *MCADParams, template string, fns // +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads/status,verbs=get;update;patch // +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads/finalizers,verbs=update -// +kubebuilder:rbac:groups=mcad.ibm.com,resources=queuejobs;schedulingspecs;appwrappers;appwrappers/finalizers;appwrappers/status,verbs=get;list;watch;create;update;patch;delete;deletecollection +// +kubebuilder:rbac:groups=workload.codeflare.dev,resources=queuejobs;schedulingspecs;appwrappers;appwrappers/finalizers;appwrappers/status,verbs=get;list;watch;create;update;patch;delete;deletecollection // +kubebuilder:rbac:groups=core,resources=pods;lists;namespaces,verbs=get;list;watch;create;update;patch;delete;deletecollection // +kubebuilder:rbac:groups=core,resources=bindings;pods/binding,verbs=create // +kubebuilder:rbac:groups=core,resources=kube-scheduler,verbs=get;update diff --git a/controllers/testdata/instascale_test_results/case_1/clusterrole.yaml b/controllers/testdata/instascale_test_results/case_1/clusterrole.yaml index c4053cdb..150863ed 100644 --- a/controllers/testdata/instascale_test_results/case_1/clusterrole.yaml +++ b/controllers/testdata/instascale_test_results/case_1/clusterrole.yaml @@ -63,6 +63,6 @@ rules: - delete - patch apiGroups: - - mcad.ibm.com + - workload.codeflare.dev resources: - appwrappers diff --git a/go.mod b/go.mod index 764faf2e..c621d70c 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230831150053-05b1af83014c github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 @@ -21,6 +21,8 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 ) +replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 + require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index b87ec39e..a87dbc9c 100644 --- a/go.sum +++ b/go.sum @@ -495,8 +495,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0 h1:4J3vW+vuCBHyocd3huZy3FJhOw5acaBHFLYwutkTE/E= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.0/go.mod h1:gtTl8Tsl+X+bGhqVudLoveINR6IkN+sVvH0J+VZIP40= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230831150053-05b1af83014c h1:LjCgRcjtkd9byePczvxGBLjdM22M2BCTB76FoEk2fkU= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230831150053-05b1af83014c/go.mod h1:gtTl8Tsl+X+bGhqVudLoveINR6IkN+sVvH0J+VZIP40= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 82f5b808..23ad3095 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -137,7 +137,7 @@ func TestMNISTPyTorchMCAD(t *testing.T) { }, } - _, err = test.Client().MCAD().McadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) + _, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index bf618cbf..413c8c5e 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -217,7 +217,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, } - aw, err = test.Client().MCAD().McadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) + aw, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) diff --git a/test/support/mcad.go b/test/support/mcad.go index a4d69b75..a5ce310f 100644 --- a/test/support/mcad.go +++ b/test/support/mcad.go @@ -26,7 +26,7 @@ import ( func AppWrapper(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *mcadv1beta1.AppWrapper { return func(g gomega.Gomega) *mcadv1beta1.AppWrapper { - aw, err := t.Client().MCAD().McadV1beta1().AppWrappers(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) + aw, err := t.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) return aw } From 4ae9e8f9e640877ad3681df7266547450da7f2ed Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 30 Aug 2023 09:48:47 +0200 Subject: [PATCH 094/377] Test support: Return AppWrappers based on prefix --- test/support/client.go | 26 ++++++++++++++++++++++++++ test/support/mcad.go | 12 ++++++++++++ 2 files changed, 38 insertions(+) diff --git a/test/support/client.go b/test/support/client.go index 86a895ee..1a02e7eb 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -20,10 +20,12 @@ import ( mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/versioned" rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/tools/clientcmd" + imagev1 "github.com/openshift/client-go/image/clientset/versioned" routev1 "github.com/openshift/client-go/route/clientset/versioned" codeflareclient "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" @@ -34,17 +36,21 @@ import ( type Client interface { Core() kubernetes.Interface Route() routev1.Interface + Image() imagev1.Interface CodeFlare() codeflareclient.Interface MCAD() mcadclient.Interface Ray() rayclient.Interface + Dynamic() dynamic.Interface } type testClient struct { core kubernetes.Interface route routev1.Interface + image imagev1.Interface codeflare codeflareclient.Interface mcad mcadclient.Interface ray rayclient.Interface + dynamic dynamic.Interface } var _ Client = (*testClient)(nil) @@ -57,6 +63,10 @@ func (t *testClient) Route() routev1.Interface { return t.route } +func (t *testClient) Image() imagev1.Interface { + return t.image +} + func (t *testClient) CodeFlare() codeflareclient.Interface { return t.codeflare } @@ -69,6 +79,10 @@ func (t *testClient) Ray() rayclient.Interface { return t.ray } +func (t *testClient) Dynamic() dynamic.Interface { + return t.dynamic +} + func newTestClient() (Client, error) { cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( clientcmd.NewDefaultClientConfigLoadingRules(), @@ -88,6 +102,11 @@ func newTestClient() (Client, error) { return nil, err } + imageClient, err := imagev1.NewForConfig(cfg) + if err != nil { + return nil, err + } + codeFlareClient, err := codeflareclient.NewForConfig(cfg) if err != nil { return nil, err @@ -103,11 +122,18 @@ func newTestClient() (Client, error) { return nil, err } + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, err + } + return &testClient{ core: kubeClient, route: routeClient, + image: imageClient, codeflare: codeFlareClient, mcad: mcadClient, ray: rayClient, + dynamic: dynamicClient, }, nil } diff --git a/test/support/mcad.go b/test/support/mcad.go index a5ce310f..d243852d 100644 --- a/test/support/mcad.go +++ b/test/support/mcad.go @@ -32,6 +32,18 @@ func AppWrapper(t Test, namespace *corev1.Namespace, name string) func(g gomega. } } +func AppWrappers(t Test, namespace *corev1.Namespace) func(g gomega.Gomega) []mcadv1beta1.AppWrapper { + return func(g gomega.Gomega) []mcadv1beta1.AppWrapper { + aws, err := t.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).List(t.Ctx(), metav1.ListOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return aws.Items + } +} + +func AppWrapperName(aw *mcadv1beta1.AppWrapper) string { + return aw.Name +} + func AppWrapperState(aw *mcadv1beta1.AppWrapper) mcadv1beta1.AppWrapperState { return aw.Status.State } From c093d58c19b34b337f0e2c59b8c5372b8b243752 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 6 Sep 2023 15:56:03 +0200 Subject: [PATCH 095/377] Use sed command to replace MCAD repo reference in Makefile --- Makefile | 10 +++++----- README.md | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 6d498b2c..6e104525 100644 --- a/Makefile +++ b/Makefile @@ -249,26 +249,26 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) + sed -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/crd | kubectl apply -f - git restore config/* .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) + sed -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - git restore config/* .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) + sed -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - git restore config/* .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) + sed -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - git restore config/* @@ -367,7 +367,7 @@ validate-bundle: install-operator-sdk .PHONY: bundle bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q - $(KUSTOMIZE) fn run config/crd/mcad --image gcr.io/kpt-fn/apply-setters:v0.2.0 -- MCAD_CRD=$(MCAD_CRD) + sed -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion diff --git a/README.md b/README.md index fe5ce178..685e85f2 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,9 @@ CodeFlare Stack Compatibility Matrix ## Development +Requirements: +- GNU sed - sed is used in several Makefile command. Using macOS default sed is incompatible, so GNU sed is needed for correct execution of these commands. + ### Testing The e2e tests can be executed locally by running the following commands: From e5ec1346c503ce5c9ce058e58129a79bc960cebe Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Wed, 6 Sep 2023 16:59:17 +0100 Subject: [PATCH 096/377] Remove external metrics APIService resources --- .../mcad/apiservice_external-metrics.yaml | 18 ------------------ config/internal/mcad/deployment.yaml.tmpl | 7 ------- config/internal/mcad/service.yaml.tmpl | 15 --------------- controllers/multi_cluster_app_dispatcher.go | 1 - 4 files changed, 41 deletions(-) delete mode 100644 config/internal/mcad/apiservice_external-metrics.yaml delete mode 100644 config/internal/mcad/service.yaml.tmpl diff --git a/config/internal/mcad/apiservice_external-metrics.yaml b/config/internal/mcad/apiservice_external-metrics.yaml deleted file mode 100644 index 0c242c11..00000000 --- a/config/internal/mcad/apiservice_external-metrics.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# {{ if (eq .Values.configMap.multiCluster true) }} -apiVersion: apiregistration.k8s.io/v1beta1 -kind: APIService -metadata: - name: v1beta1.external.metrics.k8s.io - labels: - app.kubernetes.io/managed-by: MCAD - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} -spec: - service: - name: mcad-{{.Name}}-metrics - namespace: {{.Namespace}} - group: external.metrics.k8s.io - version: v1beta1 - insecureSkipTLSVerify: true - groupPriorityMinimum: 100 - versionPriority: 100 diff --git a/config/internal/mcad/deployment.yaml.tmpl b/config/internal/mcad/deployment.yaml.tmpl index 4620c1dc..502ab69d 100644 --- a/config/internal/mcad/deployment.yaml.tmpl +++ b/config/internal/mcad/deployment.yaml.tmpl @@ -27,13 +27,6 @@ spec: name: mcad-{{.Name}}-config image: {{.ControllerImage}} imagePullPolicy: Always - ports: - - name: https - containerPort: 6443 - protocol: TCP - - name: http - containerPort: 8080 - protocol: TCP resources: {{.ControllerResources}} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File diff --git a/config/internal/mcad/service.yaml.tmpl b/config/internal/mcad/service.yaml.tmpl deleted file mode 100644 index 3bdd1555..00000000 --- a/config/internal/mcad/service.yaml.tmpl +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: mcad-{{.Name}}-metrics - namespace: {{.Namespace}} -spec: - ports: - - name: https - port: 443 - targetPort: 6443 - - name: http - port: 80 - targetPort: 8080 - selector: - app: mcad-{{.Name}} diff --git a/controllers/multi_cluster_app_dispatcher.go b/controllers/multi_cluster_app_dispatcher.go index ca3b7e79..3d80e3c5 100644 --- a/controllers/multi_cluster_app_dispatcher.go +++ b/controllers/multi_cluster_app_dispatcher.go @@ -22,7 +22,6 @@ import ( var multiClusterAppDispatcherTemplates = []string{ "mcad/configmap.yaml.tmpl", - "mcad/service.yaml.tmpl", "mcad/serviceaccount.yaml.tmpl", "mcad/deployment.yaml.tmpl", } From 8e47723cb4dec773e65080b9bba32c6f3d15e2b3 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 7 Sep 2023 12:22:18 +0100 Subject: [PATCH 097/377] Removed RBs, services, and ports from test results --- .../mcad_test_results/case_1/rolebinding.yaml | 13 ---------- .../mcad_test_results/case_1/service.yaml | 26 ------------------- .../mcad_test_results/case_2/rolebinding.yaml | 13 ---------- .../mcad_test_results/case_2/service.yaml | 26 ------------------- .../mcad_test_results/case_3/deployment.yaml | 7 ----- 5 files changed, 85 deletions(-) delete mode 100644 controllers/testdata/mcad_test_results/case_1/rolebinding.yaml delete mode 100644 controllers/testdata/mcad_test_results/case_1/service.yaml delete mode 100644 controllers/testdata/mcad_test_results/case_2/rolebinding.yaml delete mode 100644 controllers/testdata/mcad_test_results/case_2/service.yaml diff --git a/controllers/testdata/mcad_test_results/case_1/rolebinding.yaml b/controllers/testdata/mcad_test_results/case_1/rolebinding.yaml deleted file mode 100644 index d9205e2b..00000000 --- a/controllers/testdata/mcad_test_results/case_1/rolebinding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: blank-custom-resource-custom-metrics-auth-reader - namespace: default -subjects: - - kind: ServiceAccount - name: mcad-controller-blank-custom-resource - namespace: default -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader diff --git a/controllers/testdata/mcad_test_results/case_1/service.yaml b/controllers/testdata/mcad_test_results/case_1/service.yaml deleted file mode 100644 index 6b6b57e7..00000000 --- a/controllers/testdata/mcad_test_results/case_1/service.yaml +++ /dev/null @@ -1,26 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: mcad-blank-custom-resource-metrics - namespace: default -spec: - clusterIP: 172.31.181.101 - ipFamilies: - - IPv4 - ports: - - name: https - protocol: TCP - port: 443 - targetPort: 6443 - - name: http - protocol: TCP - port: 80 - targetPort: 8080 - internalTrafficPolicy: Cluster - clusterIPs: - - 172.31.181.101 - type: ClusterIP - ipFamilyPolicy: SingleStack - sessionAffinity: None - selector: - app: mcad-blank-custom-resource diff --git a/controllers/testdata/mcad_test_results/case_2/rolebinding.yaml b/controllers/testdata/mcad_test_results/case_2/rolebinding.yaml deleted file mode 100644 index 521a11b2..00000000 --- a/controllers/testdata/mcad_test_results/case_2/rolebinding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: populated-custom-resource-custom-metrics-auth-reader - namespace: default -subjects: - - kind: ServiceAccount - name: mcad-controller-populated-custom-resource - namespace: default -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader diff --git a/controllers/testdata/mcad_test_results/case_2/service.yaml b/controllers/testdata/mcad_test_results/case_2/service.yaml deleted file mode 100644 index 7f2d1d2b..00000000 --- a/controllers/testdata/mcad_test_results/case_2/service.yaml +++ /dev/null @@ -1,26 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: mcad-populated-custom-resource-metrics - namespace: default -spec: - clusterIP: 172.31.181.101 - ipFamilies: - - IPv4 - ports: - - name: https - protocol: TCP - port: 443 - targetPort: 6443 - - name: http - protocol: TCP - port: 80 - targetPort: 8080 - internalTrafficPolicy: Cluster - clusterIPs: - - 172.31.181.101 - type: ClusterIP - ipFamilyPolicy: SingleStack - sessionAffinity: None - selector: - app: mcad-populated-custom-resource diff --git a/controllers/testdata/mcad_test_results/case_3/deployment.yaml b/controllers/testdata/mcad_test_results/case_3/deployment.yaml index 5efa0df2..03f8dffd 100644 --- a/controllers/testdata/mcad_test_results/case_3/deployment.yaml +++ b/controllers/testdata/mcad_test_results/case_3/deployment.yaml @@ -27,13 +27,6 @@ spec: name: mcad-custom-image-config image: quay.io/project-codeflare/mcad-controller:custom imagePullPolicy: Always - ports: - - name: https - containerPort: 6443 - protocol: TCP - - name: http - containerPort: 8080 - protocol: TCP terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: From 2a4f5b8a6c5dc4dcf22be231999169eab87e08b7 Mon Sep 17 00:00:00 2001 From: Maxusmusti Date: Mon, 11 Sep 2023 15:29:00 +0000 Subject: [PATCH 098/377] Update dependency versions for release v0.2.3 --- Makefile | 8 ++++---- README.md | 8 ++++---- controllers/defaults.go | 4 ++-- .../instascale_test_results/case_1/deployment.yaml | 2 +- .../instascale_test_results/case_2/deployment.yaml | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- test/support/defaults.go | 2 +- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 6e104525..8f39a9bc 100644 --- a/Makefile +++ b/Makefile @@ -12,12 +12,12 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= dev +INSTASCALE_VERSION ?= v0.0.8 # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= main +MCAD_VERSION ?= v1.34.1 # MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources -MCAD_REF ?= dev +MCAD_REF ?= release-${MCAD_VERSION} MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) # The image is still published using the MCAD_REF format (i.e release-vX.Y.Z) @@ -30,7 +30,7 @@ KUBERAY_VERSION ?= v0.5.0 RAY_VERSION ?= 2.5.0 # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK -CODEFLARE_SDK_VERSION ?= 0.7.0 +CODEFLARE_SDK_VERSION ?= 0.7.1 # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to diff --git a/README.md b/README.md index 685e85f2..0f74b5b8 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------| -| CodeFlare Operator | v0.2.2 | -| Multi-Cluster App Dispatcher | v1.34.0 | -| CodeFlare-SDK | v0.7.0 | -| InstaScale | v0.0.7 | +| CodeFlare Operator | v0.2.3 | +| Multi-Cluster App Dispatcher | v1.34.1 | +| CodeFlare-SDK | v0.7.1 | +| InstaScale | v0.0.8 | | KubeRay | v0.5.0 | diff --git a/controllers/defaults.go b/controllers/defaults.go index bac326fd..9f28b3c1 100644 --- a/controllers/defaults.go +++ b/controllers/defaults.go @@ -5,6 +5,6 @@ package controllers // *********************** const ( - MCADImage = "quay.io/project-codeflare/mcad-controller:dev" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:dev" + MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.34.1" + InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.8" ) diff --git a/controllers/testdata/instascale_test_results/case_1/deployment.yaml b/controllers/testdata/instascale_test_results/case_1/deployment.yaml index 4e3c3078..4587ba19 100644 --- a/controllers/testdata/instascale_test_results/case_1/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_1/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.7 + image: quay.io/project-codeflare/instascale-controller:v0.0.8 resources: limits: cpu: '2' diff --git a/controllers/testdata/instascale_test_results/case_2/deployment.yaml b/controllers/testdata/instascale_test_results/case_2/deployment.yaml index bb8eeeea..f5b353ff 100644 --- a/controllers/testdata/instascale_test_results/case_2/deployment.yaml +++ b/controllers/testdata/instascale_test_results/case_2/deployment.yaml @@ -19,7 +19,7 @@ spec: - name: instascale args: - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.7 + image: quay.io/project-codeflare/instascale-controller:v0.0.8 resources: limits: cpu: '1' diff --git a/go.mod b/go.mod index c621d70c..07fe60ac 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,8 @@ require ( github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230831150053-05b1af83014c - github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1 + github.com/ray-project/kuberay/ray-operator v0.0.0-20230908233208-a8f730e5a2b6 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index a87dbc9c..c59002f7 100644 --- a/go.sum +++ b/go.sum @@ -495,8 +495,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230831150053-05b1af83014c h1:LjCgRcjtkd9byePczvxGBLjdM22M2BCTB76FoEk2fkU= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230831150053-05b1af83014c/go.mod h1:gtTl8Tsl+X+bGhqVudLoveINR6IkN+sVvH0J+VZIP40= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1 h1:ZNQ/JPdjS6CtaAzt6SNqaoWcpwS1PyVdgZlmIYikPLI= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -528,8 +528,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576 h1:NYt+ipGo+jMLYCFI5S04DzVFSXn8WCR0m6m3bFXPk+4= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230830082034-e7fbf7d73576/go.mod h1:7PBzsR4L0b/f5tTxq1XGnzvudbrMyPadiLiNiP9pOl0= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230908233208-a8f730e5a2b6 h1:NP7S29MtvEH4zD0oawHqGE/DXu5uYbGvXQq6mh5+3Kk= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230908233208-a8f730e5a2b6/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/test/support/defaults.go b/test/support/defaults.go index c87b5b1a..66f36484 100644 --- a/test/support/defaults.go +++ b/test/support/defaults.go @@ -5,7 +5,7 @@ package support // *********************** const ( - CodeFlareSDKVersion = "0.6.1" + CodeFlareSDKVersion = "0.7.1" RayVersion = "2.5.0" RayImage = "rayproject/ray:2.5.0" ) From af877c6085f3678d8a6c99be35dc61fa76801e55 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Mon, 21 Aug 2023 14:09:05 +0100 Subject: [PATCH 099/377] build: add scorecard-bundle make target --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 8f39a9bc..71c47e03 100644 --- a/Makefile +++ b/Makefile @@ -467,3 +467,7 @@ imports: openshift-goimports ## Organize imports in go files using openshift-goi .PHONY: verify-imports verify-imports: openshift-goimports ## Run import verifications. ./hack/verify-imports.sh $(OPENSHIFT-GOIMPORTS) + +.PHONY: scorecard-bundle +scorecard-bundle: ## Run scorecard tests on bundle image. + $(OPERATOR_SDK) scorecard bundle From ea584a84a769b5abe03090a7aa03512e6a0c43d0 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Mon, 21 Aug 2023 14:09:30 +0100 Subject: [PATCH 100/377] ci: add scorecard-bundle to ci --- .github/workflows/olm_tests.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 67c797be..058cca6c 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -134,6 +134,12 @@ jobs: BUNDLE_PUSH_OPT: "--tls-verify=false" CATALOG_PUSH_OPT: "--tls-verify=false" + - name: Scorecard check + run: | + make scorecard-bundle + env: + IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator:v0.0.1" + - name: Update Operator to the built version run: | ORIGINAL_POD_NAME=$(kubectl get pod -l app.kubernetes.io/name=codeflare-operator -n openshift-operators -o json | jq -r .items[].metadata.name) From ff67b155f6def9323156ffb27fc8e1686df0f28a Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Mon, 21 Aug 2023 14:35:40 +0100 Subject: [PATCH 101/377] refactor: removing a test which has a bug in the operator-sdk --- .github/workflows/olm_tests.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 058cca6c..eb643b13 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -134,6 +134,15 @@ jobs: BUNDLE_PUSH_OPT: "--tls-verify=false" CATALOG_PUSH_OPT: "--tls-verify=false" + - name: Install yq + run: | + sudo wget -O /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64 + sudo chmod +x /usr/bin/yq + + - name: Modify scorecard config + run: | + yq e 'del(.stages[].tests[] | select(.labels.test == "olm-crds-have-resources-test"))' -i bundle/tests/scorecard/config.yaml + - name: Scorecard check run: | make scorecard-bundle From 715344526c4ba85bd47bf004bc878d8c38374b5f Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Thu, 24 Aug 2023 13:57:34 +0100 Subject: [PATCH 102/377] ci: addition of scorecard in CI --- .github/workflows/olm_scorecard.yml | 28 ++++++++++++++++++++++++++++ .github/workflows/olm_tests.yaml | 15 --------------- Makefile | 2 +- 3 files changed, 29 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/olm_scorecard.yml diff --git a/.github/workflows/olm_scorecard.yml b/.github/workflows/olm_scorecard.yml new file mode 100644 index 00000000..ba83b801 --- /dev/null +++ b/.github/workflows/olm_scorecard.yml @@ -0,0 +1,28 @@ +name: Operator Scorecard Test + +on: [push, pull_request] + +jobs: + scorecard-check: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Setup and start KinD cluster + uses: ./.github/actions/kind + + - name: Build bundle + run: make bundle-build + + - name: Install yq + run: | + sudo wget -O /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64 + sudo chmod +x /usr/bin/yq + + - name: Modify scorecard config + run: | + yq e 'del(.stages[].tests[] | select(.labels.test == "olm-crds-have-resources-test"))' -i bundle/tests/scorecard/config.yaml + + - name: Run Operator SDK Scorecard + run: make scorecard-bundle diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index eb643b13..67c797be 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -134,21 +134,6 @@ jobs: BUNDLE_PUSH_OPT: "--tls-verify=false" CATALOG_PUSH_OPT: "--tls-verify=false" - - name: Install yq - run: | - sudo wget -O /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64 - sudo chmod +x /usr/bin/yq - - - name: Modify scorecard config - run: | - yq e 'del(.stages[].tests[] | select(.labels.test == "olm-crds-have-resources-test"))' -i bundle/tests/scorecard/config.yaml - - - name: Scorecard check - run: | - make scorecard-bundle - env: - IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator:v0.0.1" - - name: Update Operator to the built version run: | ORIGINAL_POD_NAME=$(kubectl get pod -l app.kubernetes.io/name=codeflare-operator -n openshift-operators -o json | jq -r .items[].metadata.name) diff --git a/Makefile b/Makefile index 71c47e03..e1ceb525 100644 --- a/Makefile +++ b/Makefile @@ -469,5 +469,5 @@ verify-imports: openshift-goimports ## Run import verifications. ./hack/verify-imports.sh $(OPENSHIFT-GOIMPORTS) .PHONY: scorecard-bundle -scorecard-bundle: ## Run scorecard tests on bundle image. +scorecard-bundle: install-operator-sdk ## Run scorecard tests on bundle image. $(OPERATOR_SDK) scorecard bundle From fa17d54af19d371befc4e1bcb40cef92cf58ad1b Mon Sep 17 00:00:00 2001 From: James Busche Date: Mon, 11 Sep 2023 14:33:31 -0700 Subject: [PATCH 103/377] adjust rbac to match mcad Signed-off-by: James Busche --- config/rbac/role.yaml | 36 ++++++++++++++++++---------------- controllers/mcad_controller.go | 2 +- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 691c9dfb..a2cc83f9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -270,6 +270,25 @@ rules: - get - list - watch +- apiGroups: + - quota.codeflare.dev + - workload.codeflare.dev + resources: + - appwrappers + - appwrappers/finalizers + - appwrappers/status + - queuejobs + - quotasubtrees + - schedulingspecs + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch - apiGroups: - rbac.authorization.k8s.io resources: @@ -318,20 +337,3 @@ rules: - get - list - watch -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers - - appwrappers/finalizers - - appwrappers/status - - queuejobs - - schedulingspecs - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch diff --git a/controllers/mcad_controller.go b/controllers/mcad_controller.go index 9dfecc24..ce32f08b 100644 --- a/controllers/mcad_controller.go +++ b/controllers/mcad_controller.go @@ -102,7 +102,7 @@ func (r *MCADReconciler) DeleteResource(params *MCADParams, template string, fns // +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads/status,verbs=get;update;patch // +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads/finalizers,verbs=update -// +kubebuilder:rbac:groups=workload.codeflare.dev,resources=queuejobs;schedulingspecs;appwrappers;appwrappers/finalizers;appwrappers/status,verbs=get;list;watch;create;update;patch;delete;deletecollection +// +kubebuilder:rbac:groups=workload.codeflare.dev;quota.codeflare.dev,resources=queuejobs;schedulingspecs;appwrappers;appwrappers/finalizers;appwrappers/status;quotasubtrees,verbs=get;list;watch;create;update;patch;delete;deletecollection // +kubebuilder:rbac:groups=core,resources=pods;lists;namespaces,verbs=get;list;watch;create;update;patch;delete;deletecollection // +kubebuilder:rbac:groups=core,resources=bindings;pods/binding,verbs=create // +kubebuilder:rbac:groups=core,resources=kube-scheduler,verbs=get;update From 86b35a4f1626788e7779a2b2497e4209d1aac76c Mon Sep 17 00:00:00 2001 From: James Busche Date: Tue, 12 Sep 2023 10:34:12 -0700 Subject: [PATCH 104/377] add quota items to the clusterrole Signed-off-by: James Busche --- config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl b/config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl index 49dfe097..8812cfc3 100644 --- a/config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl +++ b/config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl @@ -11,10 +11,12 @@ metadata: codeflare.codeflare.dev/cr-namespace: {{.Namespace}} rules: - apiGroups: + - quota.codeflare.dev - workload.codeflare.dev resources: - queuejobs - schedulingspecs + - quotasubtrees - appwrappers - appwrappers/finalizers - appwrappers/status From 526bb53289e1c49bac53894fee451a7b33ec719d Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 13 Sep 2023 14:30:24 +0200 Subject: [PATCH 105/377] e2e support: return AppWrapper pointers instead of structs --- test/support/mcad.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/support/mcad.go b/test/support/mcad.go index d243852d..8fdda165 100644 --- a/test/support/mcad.go +++ b/test/support/mcad.go @@ -32,11 +32,18 @@ func AppWrapper(t Test, namespace *corev1.Namespace, name string) func(g gomega. } } -func AppWrappers(t Test, namespace *corev1.Namespace) func(g gomega.Gomega) []mcadv1beta1.AppWrapper { - return func(g gomega.Gomega) []mcadv1beta1.AppWrapper { +func AppWrappers(t Test, namespace *corev1.Namespace) func(g gomega.Gomega) []*mcadv1beta1.AppWrapper { + return func(g gomega.Gomega) []*mcadv1beta1.AppWrapper { aws, err := t.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).List(t.Ctx(), metav1.ListOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) - return aws.Items + + awsp := []*mcadv1beta1.AppWrapper{} + for _, v := range aws.Items { + v := v + awsp = append(awsp, &v) + } + + return awsp } } From a14914da37d61dd2e3de160c7033e3b2fe6d500e Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Thu, 14 Sep 2023 13:58:22 +0100 Subject: [PATCH 106/377] docs: spelling correction --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 79cae2f0..83edd2f9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,7 @@ The MCAD and InstaScale custom resources are defined under the `api` dir: - See `mcad_types.go` and `instascale_types.go` The MCAD and InstaScale resource templates can be found under `config/internal`: - - Sorted under `mcad` and `insascale` subdirs + - Sorted under `mcad` and `instascale` subdirs The code for MCAD/InstaScale resource reconcilliation can be found in the `controllers` dir: - See `mcad_controller.go` and `instascale_controller.go` From 5580d52dd4f0f7ab03ad6de7c335179f7dc40b80 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 15 Sep 2023 13:45:21 +0200 Subject: [PATCH 107/377] Update to KubeRay v0.6.0 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e1ceb525..d6acd39b 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v0.5.0 +KUBERAY_VERSION ?= v0.6.0 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 From 9b2d77c378842a31b965ea4d2a5a5d18328c48c8 Mon Sep 17 00:00:00 2001 From: Jiri Petrlik Date: Thu, 14 Sep 2023 09:04:08 +0200 Subject: [PATCH 108/377] [262] Add links to other release notes --- .github/workflows/tag-and-build.yml | 8 ++++---- README.md | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 26667caa..1a322d70 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -85,10 +85,10 @@ jobs: - name: Adjust Compatibility Matrix in readme run: | - sed -i -E "s/(.*CodeFlare Operator.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.version }}\2/" README.md - sed -i -E "s/(.*Multi-Cluster App Dispatcher.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2/" README.md - sed -i -E "s/(.*CodeFlare-SDK.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.codeflare-sdk-version }}\2/" README.md - sed -i -E "s/(.*InstaScale.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2/" README.md + sed -i -E "s/(.*CodeFlare Operator.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.version }}\2${{ github.event.inputs.version }}\3/" README.md + sed -i -E "s/(.*Multi-Cluster App Dispatcher.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2${{ github.event.inputs.mcad-version }}\3/" README.md + sed -i -E "s/(.*CodeFlare-SDK.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3/" README.md + sed -i -E "s/(.*InstaScale.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2${{ github.event.inputs.instascale-version }}\3/" README.md - name: Adjust MCAD, SDK and InstaScale dependencies in the code run: | diff --git a/README.md b/README.md index 0f74b5b8..9924ee3b 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ Operator for installation and lifecycle management of CodeFlare distributed work CodeFlare Stack Compatibility Matrix -| Component | Version | -|------------------------------|---------| -| CodeFlare Operator | v0.2.3 | -| Multi-Cluster App Dispatcher | v1.34.1 | -| CodeFlare-SDK | v0.7.1 | -| InstaScale | v0.0.8 | -| KubeRay | v0.5.0 | +| Component | Version | +|------------------------------|---------------------------------------------------------------------------------------------------| +| CodeFlare Operator | [v0.2.3](https://github.com/project-codeflare/codeflare-operator/releases/tag/v0.2.3) | +| Multi-Cluster App Dispatcher | [v1.34.1](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.34.1) | +| CodeFlare-SDK | [v0.7.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.7.1) | +| InstaScale | [v0.0.8](https://github.com/project-codeflare/instascale/releases/tag/v0.0.8) | +| KubeRay | [v0.5.0](https://github.com/ray-project/kuberay/releases/tag/v0.5.0) | ## Development From cf41109d69f43858c4cf322c68bb66f688143fc0 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 8 Aug 2023 14:51:12 +0200 Subject: [PATCH 109/377] Embed MCAD and InstaScale controllers --- Makefile | 6 +- go.mod | 59 +++++++++++- go.sum | 272 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- main.go | 87 +++++++++++++----- 4 files changed, 386 insertions(+), 38 deletions(-) diff --git a/Makefile b/Makefile index d6acd39b..e0203d27 100644 --- a/Makefile +++ b/Makefile @@ -171,7 +171,7 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./api/..." .PHONY: generate-client ## Generate client packages and organize the goimports generate-client: generate-client-files imports @@ -222,8 +222,8 @@ vet: ## Run go vet against code. .PHONY: modules modules: ## Update Go dependencies. - go get $(MCAD_REPO)@$(MCAD_VERSION) - go get github.com/ray-project/kuberay/ray-operator + #go get $(MCAD_REPO)@$(MCAD_VERSION) + #go get github.com/ray-project/kuberay/ray-operator .PHONY: build build: modules defaults generate fmt vet ## Build manager binary. diff --git a/go.mod b/go.mod index 07fe60ac..75905947 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,8 @@ require ( github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1 + github.com/project-codeflare/instascale v0.0.0-00010101000000-000000000000 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230904071404-ce734452edc6 github.com/ray-project/kuberay/ray-operator v0.0.0-20230908233208-a8f730e5a2b6 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 @@ -21,42 +22,65 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 ) -replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 +replace ( + github.com/project-codeflare/instascale => github.com/Bobbins228/instascale v0.0.0-20230724130835-2a8ebf58690f + sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 +) require ( + github.com/NYTimes/gziphandler v1.1.1 // indirect + github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/eapache/go-resiliency v1.3.0 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.4.1 // indirect + github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/cel-go v0.12.6 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.1.2 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/css v1.0.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.4 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/microcosm-cc/bluemonday v1.0.18 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/openshift-online/ocm-sdk-go v0.1.327 // indirect github.com/pelletier/go-toml v1.8.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect @@ -65,16 +89,32 @@ require ( github.com/prometheus/procfs v0.8.0 // indirect github.com/spf13/afero v1.4.1 // indirect github.com/spf13/cast v1.3.1 // indirect - github.com/spf13/cobra v1.6.0 // indirect + github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.7.1 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect + go.etcd.io/etcd/api/v3 v3.5.5 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect + go.etcd.io/etcd/client/v3 v3.5.5 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 // indirect + go.opentelemetry.io/otel v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect + go.opentelemetry.io/otel/metric v0.31.0 // indirect + go.opentelemetry.io/otel/sdk v1.10.0 // indirect + go.opentelemetry.io/otel/trace v1.10.0 // indirect + go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect + golang.org/x/crypto v0.11.0 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect + golang.org/x/sync v0.2.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/term v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect @@ -82,16 +122,25 @@ require ( golang.org/x/tools v0.9.3 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect + google.golang.org/grpc v1.49.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.26.1 // indirect + k8s.io/apiserver v0.26.2 // indirect k8s.io/component-base v0.26.2 // indirect + k8s.io/klog v1.0.0 // indirect k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/kms v0.26.2 // indirect k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d // indirect + k8s.io/metrics v0.26.2 // indirect k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 // indirect + sigs.k8s.io/custom-metrics-apiserver v0.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index c59002f7..a95f13de 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,7 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.97.0 h1:3DXvAyifywvq64LfkKaMOmkWPS1CikIQdMe2lY9vxU8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -53,9 +54,15 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/Bobbins228/instascale v0.0.0-20230724130835-2a8ebf58690f h1:aEtM6OZ2dCY97VnVsVYZl2GJOe+6WegPGLgLXQt5BGk= +github.com/Bobbins228/instascale v0.0.0-20230724130835-2a8ebf58690f/go.mod h1:kwBkC+o4W+a1dpftXbVQjvGCVyu2206HlZtcp1+ZjyY= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -70,12 +77,16 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= +github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -85,7 +96,11 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -97,16 +112,26 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -124,7 +149,10 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= +github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -135,7 +163,10 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= @@ -149,8 +180,11 @@ github.com/evanphx/json-patch/v5 v5.2.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2Vvl github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -179,6 +213,8 @@ github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= @@ -232,15 +268,22 @@ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+ github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= +github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -277,7 +320,10 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.12.6 h1:kjeKudqV0OygrAqA9fX6J55S8gj+Jre2tckIm5RoG4M= +github.com/google/cel-go v0.12.6/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -290,6 +336,7 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= @@ -306,13 +353,15 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= @@ -323,24 +372,35 @@ github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97Dwqy github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -364,10 +424,63 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= +github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= +github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.12.0 h1:/RvQ24k3TnNdfBSW0ou9EOi5jx2cX7zfE8n2nLKuiP0= +github.com/jackc/pgconn v1.12.0/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL8Y= +github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs= +github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.16.0 h1:4k1tROTJctHotannFYzu77dY3bgtMRymQP7tXQjqpPk= +github.com/jackc/pgx/v4 v4.16.0/go.mod h1:N0A9sFdWzkw/Jy1lwoiB64F2+ugFZi987zRxcPez/wI= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -390,6 +503,7 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -399,9 +513,15 @@ github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQSeYtvkY= @@ -420,13 +540,22 @@ github.com/manifestival/manifestival v0.6.0/go.mod h1:3Qq9cnPy7sv7FVhg2Kvw0ebb35 github.com/manifestival/manifestival v0.7.2 h1:l4uFdWX/xQK4QcRfqGoMtBvaZeWPEuwD6hVsCwUqZY4= github.com/manifestival/manifestival v0.7.2/go.mod h1:nl3T6HlfHCeidooWVTMI9vYNTBkQ1GdhLNb+smozbdk= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= +github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -456,6 +585,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -464,6 +595,10 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -472,10 +607,14 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 h1:Zq1BYSO2UmZuu/O1tpYIaC/7ir7ljFqdEY90TwqlseI= github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2/go.mod h1:VV07Ee+14Mjpwh/ZxtwgAieiCvZ20YVUB9jLHixzswk= +github.com/openshift-online/ocm-sdk-go v0.1.327 h1:WR822bGdQoMuZ2+dFdhZz3fpD2NlJhGr+F3FJPXvqFU= +github.com/openshift-online/ocm-sdk-go v0.1.327/go.mod h1:KYOw8kAKAHyPrJcQoVR82CneQ4ofC02Na4cXXaTq4Nw= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= @@ -495,13 +634,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1 h1:ZNQ/JPdjS6CtaAzt6SNqaoWcpwS1PyVdgZlmIYikPLI= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230904071404-ce734452edc6 h1:YT6C0Z84TgftYcoNUd/dwvXxGJvXy67EeWAMQjpKvdU= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230904071404-ce734452edc6/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= @@ -533,20 +673,29 @@ github.com/ray-project/kuberay/ray-operator v0.0.0-20230908233208-a8f730e5a2b6/g github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -557,8 +706,8 @@ github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= -github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -570,6 +719,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -591,22 +741,39 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5 h1:Gqga3zA9tdAcfqobUGjSoCob5L3f8Dt5EuOp3ihNZko= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= +go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= +go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= +go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= +go.etcd.io/etcd/client/v2 v2.305.5 h1:DktRP60//JJpnPC0VBymAN/7V71GHMdjDCBt4ZPXDjI= +go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= +go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= +go.etcd.io/etcd/pkg/v3 v3.5.5 h1:Ablg7T7OkR+AeeeU32kdVhw/AGDsitkKPl7aW73ssjU= +go.etcd.io/etcd/raft/v3 v3.5.5 h1:Ibz6XyZ60OYyRopu73lLM/P+qco3YtlZMOhnXNS051I= +go.etcd.io/etcd/server/v3 v3.5.5 h1:jNjYm/9s+f9A9r6+SC4RvNaz6AqixpOvhrFdT0PvIj0= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -615,22 +782,47 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 h1:Ajldaqhxqw/gNzQA45IKFWLdG7jZuXX/wBW1d5qvbUI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= +go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= +go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 h1:TaB+1rQhddO1sF71MpZOZAuSPW1klK2M8XxfrBMfK7Y= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 h1:pDDYmo0QadUPal5fwXoY1pmMpFcdyhXOmL5drCrI3vU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 h1:KtiUEhQmj/Pa874bVYKGNVdq8NPKiacPbaRRtgXi+t4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= +go.opentelemetry.io/otel/metric v0.31.0 h1:6SiklT+gfWAwWUR0meEMxQBtihpiEs4c+vL9spDTqUs= +go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= +go.opentelemetry.io/otel/sdk v1.10.0 h1:jZ6K7sVn04kk/3DNUdJ4mqRlGDiXAVuIG+MMENpTNdY= +go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= +go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= +go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= @@ -639,6 +831,7 @@ golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -648,7 +841,14 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -671,6 +871,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -679,6 +880,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -724,12 +927,18 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -739,6 +948,7 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= @@ -752,6 +962,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -761,8 +974,10 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -771,6 +986,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -782,6 +998,7 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -801,15 +1018,22 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -826,6 +1050,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -852,6 +1077,7 @@ golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -861,6 +1087,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -874,6 +1101,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -894,9 +1122,14 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -959,7 +1192,11 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -975,7 +1212,13 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -989,6 +1232,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1001,15 +1245,18 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1050,6 +1297,8 @@ k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0 k8s.io/apimachinery v0.26.3 h1:dQx6PNETJ7nODU3XPtrwkfuubs6w7sX0M8n61zHIV/k= k8s.io/apimachinery v0.26.3/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= +k8s.io/apiserver v0.26.2 h1:Pk8lmX4G14hYqJd1poHGC08G03nIHVqdJMR0SD3IH3o= +k8s.io/apiserver v0.26.2/go.mod h1:GHcozwXgXsPuOJ28EnQ/jXEM9QeG6HT22YxSNmpYNh8= k8s.io/client-go v0.15.7/go.mod h1:QMNB76d3lKPvPQdOOnnxUF693C3hnCzUbC2umg70pWA= k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= @@ -1064,6 +1313,7 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= @@ -1071,12 +1321,16 @@ k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kms v0.26.2 h1:GM1gg3tFK3OUU/QQFi93yGjG3lJT8s8l3Wkn2+VxBLM= +k8s.io/kms v0.26.2/go.mod h1:69qGnf1NsFOQP07fBYqNLZklqEHSJF024JqYCaeVxHg= k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kube-openapi v0.0.0-20200204173128-addea2498afe/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d h1:VcFq5n7wCJB2FQMCIHfC+f+jNcGgNMar1uKd6rVlifU= k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= +k8s.io/metrics v0.26.2 h1:2gUvUWWnHPdE2tyA5DvyHC8HGryr+izhY9i5dzLP06s= +k8s.io/metrics v0.26.2/go.mod h1:PX1wm9REV9hSGuw9GcXTFNDgab1KRXck3mNeiLYbRho= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -1087,9 +1341,13 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 h1:+xBL5uTc+BkPBwmMi3vYfUJjq+N3K+H6PXeETwf5cPI= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo= sigs.k8s.io/controller-runtime v0.7.2/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= +sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 h1:puQ5YlyBjhxg+OQ1YPMJXwtk7WhC4E6AlWIQ9pC8jws= +sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3/go.mod h1:9nUXR/EgdYZto1aQ6yhwOksPR7J979jSyOqic1IgaOo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/main.go b/main.go index 51d4dbc9..01593b9b 100644 --- a/main.go +++ b/main.go @@ -21,6 +21,9 @@ import ( "os" "time" + instascale "github.com/project-codeflare/instascale/controllers" + mcadoptions "github.com/project-codeflare/multi-cluster-app-dispatcher/cmd/kar-controllers/app/options" + mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob" "go.uber.org/zap/zapcore" "k8s.io/apimachinery/pkg/runtime" @@ -32,16 +35,15 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - "github.com/project-codeflare/codeflare-operator/controllers" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. // +kubebuilder:scaffold:imports ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") - templatesPath = "config/internal/" + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + // templatesPath = "config/internal/" ) func init() { @@ -55,19 +57,35 @@ func main() { var metricsAddr string var enableLeaderElection bool var probeAddr string + var configsNamespace string + var ocmSecretNamespace string + + // Operator flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - opts := zap.Options{ + + // InstScale + flag.StringVar(&configsNamespace, "configs-namespace", "kube-system", "The namespace containing the Instacale configmap") + flag.StringVar(&ocmSecretNamespace, "ocm-secret-namespace", "default", "The namespace containing the OCM secret") + + mcadOptions := mcadoptions.NewServerOption() + + zapOptions := zap.Options{ Development: true, TimeEncoder: zapcore.TimeEncoderOfLayout(time.RFC3339), } - opts.BindFlags(flag.CommandLine) + + flagSet := flag.CommandLine + zapOptions.BindFlags(flagSet) + mcadOptions.AddFlags(flagSet) flag.Parse() - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) + + ctx := ctrl.SetupSignalHandler() mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, @@ -82,24 +100,40 @@ func main() { os.Exit(1) } - if err = (&controllers.MCADReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: ctrl.Log, - TemplatesPath: templatesPath, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "MCAD") + mcadQueueController := mcad.NewJobController(mgr.GetConfig(), mcadOptions) + if mcadQueueController == nil { + // FIXME: update NewJobController so it follows Go idiomatic error handling and return an error instead of a nil object os.Exit(1) } - if err = (&controllers.InstaScaleReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: ctrl.Log, - TemplatesPath: templatesPath, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "InstaScale") - os.Exit(1) + mcadQueueController.Run(ctx.Done()) + + instascaleController := &instascale.AppWrapperReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ConfigsNamespace: configsNamespace, + OcmSecretNamespace: ocmSecretNamespace, } + + exitOnError(instascaleController.SetupWithManager(mgr), "Error setting up InstaScale controller") + + // if err = (&controllers.MCADReconciler{ + // Client: mgr.GetClient(), + // Scheme: mgr.GetScheme(), + // Log: ctrl.Log, + // TemplatesPath: templatesPath, + // }).SetupWithManager(mgr); err != nil { + // setupLog.Error(err, "unable to create controller", "controller", "MCAD") + // os.Exit(1) + // } + // if err = (&controllers.InstaScaleReconciler{ + // Client: mgr.GetClient(), + // Scheme: mgr.GetScheme(), + // Log: ctrl.Log, + // TemplatesPath: templatesPath, + // }).SetupWithManager(mgr); err != nil { + // setupLog.Error(err, "unable to create controller", "controller", "InstaScale") + // os.Exit(1) + // } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { @@ -112,8 +146,15 @@ func main() { } setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + if err := mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } } + +func exitOnError(err error, msg string) { + if err != nil { + setupLog.Error(err, msg) + os.Exit(1) + } +} From a4ce80c911a3e9450bd25d5f1b4cc9052b59848f Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 10 Aug 2023 20:27:30 +0200 Subject: [PATCH 110/377] Increase golangci-lint timeout --- .golangci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yaml b/.golangci.yaml index 573c8291..e2afb25b 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,5 +1,5 @@ run: - timeout: 5m + timeout: 10m linters: enable: - errcheck From cf9020b74eabfb441d11f909edfa2df514a9dd73 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 11 Aug 2023 12:55:43 +0200 Subject: [PATCH 111/377] fix duplicated kubeconfig CLI option --- main.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 01593b9b..f75fd1bd 100644 --- a/main.go +++ b/main.go @@ -78,10 +78,16 @@ func main() { TimeEncoder: zapcore.TimeEncoderOfLayout(time.RFC3339), } - flagSet := flag.CommandLine + flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) + flag.CommandLine.VisitAll(func(f *flag.Flag) { + if f.Name != "kubeconfig" { + flagSet.Var(f.Value, f.Name, f.Usage) + } + }) + zapOptions.BindFlags(flagSet) mcadOptions.AddFlags(flagSet) - flag.Parse() + _ = flagSet.Parse(os.Args[1:]) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) From 614955ea32e6c07e82ef4cfc4d20cf9b7640bcab Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 11 Aug 2023 12:57:04 +0200 Subject: [PATCH 112/377] Add MCAD APIs to runtime scheme --- main.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index f75fd1bd..8a8111c6 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,7 @@ import ( instascale "github.com/project-codeflare/instascale/controllers" mcadoptions "github.com/project-codeflare/multi-cluster-app-dispatcher/cmd/kar-controllers/app/options" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob" "go.uber.org/zap/zapcore" @@ -33,8 +34,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. // +kubebuilder:scaffold:imports @@ -48,9 +47,7 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - - utilruntime.Must(codeflarev1alpha1.AddToScheme(scheme)) - // +kubebuilder:scaffold:scheme + _ = mcadv1beta1.AddToScheme(scheme) } func main() { From 24129e3310c8ddc05e184407a46308b33a5090ea Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 11 Aug 2023 13:07:01 +0200 Subject: [PATCH 113/377] Add InstaScale ConfigMap --- config/manager/instascale-configmap.yaml | 6 ++++++ config/manager/kustomization.yaml | 1 + config/manager/manager.yaml | 1 + 3 files changed, 8 insertions(+) create mode 100644 config/manager/instascale-configmap.yaml diff --git a/config/manager/instascale-configmap.yaml b/config/manager/instascale-configmap.yaml new file mode 100644 index 00000000..9ff6c095 --- /dev/null +++ b/config/manager/instascale-configmap.yaml @@ -0,0 +1,6 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: instascale-config +data: + maxScaleoutAllowed: "15" diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 538c69f2..85d2c251 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,5 +1,6 @@ resources: - manager.yaml +- instascale-configmap.yaml generatorOptions: disableNameSuffixHash: true diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index a8467ccf..efb31870 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -38,6 +38,7 @@ spec: - "--health-probe-bind-address=:8081" - "--metrics-bind-address=0.0.0.0:8080" - "--leader-elect" + - "--configs-namespace=codeflare-system" image: controller:latest imagePullPolicy: Always name: manager From 3511d18f48f1f881a56356d076c2eb51ee495f1a Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 11 Aug 2023 13:10:38 +0200 Subject: [PATCH 114/377] Update e2e setup --- test/e2e/setup.sh | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index dbc973dd..6940a701 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -22,16 +22,6 @@ kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/conf kubectl create ns codeflare-system --dry-run=client -o yaml | kubectl apply -f - -echo Deploying MCAD controller -cat < Date: Fri, 11 Aug 2023 13:53:35 +0200 Subject: [PATCH 115/377] Remove MCAD logs retrieval --- .github/workflows/e2e_tests.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 236883ea..42ee0b73 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -97,12 +97,6 @@ jobs: echo "Printing CodeFlare operator logs" kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log - - name: Print MCAD controller logs - if: always() && steps.deploy.outcome == 'success' - run: | - echo "Printing MCAD controller logs" - kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log - - name: Print KubeRay operator logs if: always() && steps.deploy.outcome == 'success' run: | From 82a3277b206fc846c071ab788db3d1c4745ff991 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 11 Aug 2023 15:44:57 +0200 Subject: [PATCH 116/377] Work-around conflict with MCAD custom metrics adapter port --- config/manager/controller_manager_config.yaml | 2 +- config/manager/manager.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml index 7741092e..21133591 100644 --- a/config/manager/controller_manager_config.yaml +++ b/config/manager/controller_manager_config.yaml @@ -3,7 +3,7 @@ kind: ControllerManagerConfig health: healthProbeBindAddress: :8081 metrics: - bindAddress: 0.0.0.0:8080 + bindAddress: 0.0.0.0:8082 webhook: port: 9443 leaderElection: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index efb31870..c92b8ffc 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -36,7 +36,7 @@ spec: - /manager args: - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=0.0.0.0:8080" + - "--metrics-bind-address=0.0.0.0:8082" - "--leader-elect" - "--configs-namespace=codeflare-system" image: controller:latest From 8736d6cfe7fb149f2b5546f4a48474a96fb323a9 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 11 Aug 2023 16:03:11 +0200 Subject: [PATCH 117/377] Fix InstaScale ConfigMap configuration --- .github/workflows/e2e_tests.yaml | 3 +++ config/{manager => default}/instascale-configmap.yaml | 0 config/default/kustomization.yaml | 2 ++ config/manager/kustomization.yaml | 1 - config/manager/manager.yaml | 2 +- 5 files changed, 6 insertions(+), 2 deletions(-) rename config/{manager => default}/instascale-configmap.yaml (100%) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 42ee0b73..6abb3e84 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -73,6 +73,9 @@ jobs: echo Deploying CodeFlare operator IMG="${REGISTRY_ADDRESS}"/codeflare-operator make image-push -e IMG="${IMG}" + # FIXME + kubectl create ns openshift-operators + kubectl apply -n openshift-operators -f config/default/instascale-configmap.yaml make deploy -e IMG="${IMG}" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager diff --git a/config/manager/instascale-configmap.yaml b/config/default/instascale-configmap.yaml similarity index 100% rename from config/manager/instascale-configmap.yaml rename to config/default/instascale-configmap.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index ad203a94..d3937269 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -28,6 +28,8 @@ bases: resources: # Add metrics service - metrics_service.yaml +# TODO: refactor configuration +- instascale-configmap.yaml # Mount the controller config file for loading manager configurations # through a ComponentConfig type diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 85d2c251..538c69f2 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,6 +1,5 @@ resources: - manager.yaml -- instascale-configmap.yaml generatorOptions: disableNameSuffixHash: true diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index c92b8ffc..519ecc1c 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -38,7 +38,7 @@ spec: - "--health-probe-bind-address=:8081" - "--metrics-bind-address=0.0.0.0:8082" - "--leader-elect" - - "--configs-namespace=codeflare-system" + - "--configs-namespace=openshift-operators" image: controller:latest imagePullPolicy: Always name: manager From 04c913743752186bed79112a907a4182c69c2dfe Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 11 Aug 2023 17:37:11 +0200 Subject: [PATCH 118/377] Fix RayCluster RBAC --- test/e2e/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index 6940a701..95a505d8 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -51,8 +51,8 @@ metadata: name: mcad-controller-rayclusters subjects: - kind: ServiceAccount - name: controller-manager - namespace: codeflare-system + name: codeflare-operator-controller-manager + namespace: openshift-operators roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole From b96c4bdb9892c84d5a5b784e799b3db1cb804aef Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 11 Aug 2023 18:01:51 +0200 Subject: [PATCH 119/377] Remove deployment controllers --- Dockerfile | 1 - Makefile | 17 +- controllers/config/manifest.go | 18 -- controllers/config/templating.go | 48 ---- controllers/defaults.go | 10 - controllers/instascale.go | 26 -- controllers/instascale_controller.go | 240 ---------------- controllers/instascale_controller_test.go | 66 ----- controllers/instascale_params.go | 61 ---- controllers/mcad_controller.go | 247 ----------------- controllers/mcad_controller_test.go | 66 ----- controllers/mcad_params.go | 57 ---- controllers/multi_cluster_app_dispatcher.go | 66 ----- controllers/suite_test.go | 262 ------------------ .../instascale_test_cases/case_1.yaml | 5 - .../instascale_test_cases/case_2.yaml | 12 - .../instascale_test_cases/case_3.yaml | 6 - .../case_1/clusterrole.yaml | 68 ----- .../case_1/clusterrolebinding.yaml | 12 - .../case_1/configmap.yaml | 11 - .../case_1/deployment.yaml | 30 -- .../case_1/serviceaccount.yaml | 5 - .../case_2/deployment.yaml | 30 -- .../case_3/deployment.yaml | 30 -- .../testdata/mcad_test_cases/case_1.yaml | 5 - .../testdata/mcad_test_cases/case_2.yaml | 15 - .../testdata/mcad_test_cases/case_3.yaml | 6 - .../mcad_test_results/case_1/configmap.yaml | 11 - .../case_1/serviceaccount.yaml | 5 - .../mcad_test_results/case_2/configmap.yaml | 14 - .../case_2/serviceaccount.yaml | 5 - .../mcad_test_results/case_3/deployment.yaml | 38 --- controllers/util/util.go | 183 ------------ main.go | 20 -- 34 files changed, 1 insertion(+), 1695 deletions(-) delete mode 100644 controllers/config/manifest.go delete mode 100644 controllers/config/templating.go delete mode 100644 controllers/defaults.go delete mode 100644 controllers/instascale.go delete mode 100644 controllers/instascale_controller.go delete mode 100644 controllers/instascale_controller_test.go delete mode 100644 controllers/instascale_params.go delete mode 100644 controllers/mcad_controller.go delete mode 100644 controllers/mcad_controller_test.go delete mode 100644 controllers/mcad_params.go delete mode 100644 controllers/multi_cluster_app_dispatcher.go delete mode 100644 controllers/suite_test.go delete mode 100644 controllers/testdata/instascale_test_cases/case_1.yaml delete mode 100644 controllers/testdata/instascale_test_cases/case_2.yaml delete mode 100644 controllers/testdata/instascale_test_cases/case_3.yaml delete mode 100644 controllers/testdata/instascale_test_results/case_1/clusterrole.yaml delete mode 100644 controllers/testdata/instascale_test_results/case_1/clusterrolebinding.yaml delete mode 100644 controllers/testdata/instascale_test_results/case_1/configmap.yaml delete mode 100644 controllers/testdata/instascale_test_results/case_1/deployment.yaml delete mode 100644 controllers/testdata/instascale_test_results/case_1/serviceaccount.yaml delete mode 100644 controllers/testdata/instascale_test_results/case_2/deployment.yaml delete mode 100644 controllers/testdata/instascale_test_results/case_3/deployment.yaml delete mode 100644 controllers/testdata/mcad_test_cases/case_1.yaml delete mode 100644 controllers/testdata/mcad_test_cases/case_2.yaml delete mode 100644 controllers/testdata/mcad_test_cases/case_3.yaml delete mode 100644 controllers/testdata/mcad_test_results/case_1/configmap.yaml delete mode 100644 controllers/testdata/mcad_test_results/case_1/serviceaccount.yaml delete mode 100644 controllers/testdata/mcad_test_results/case_2/configmap.yaml delete mode 100644 controllers/testdata/mcad_test_results/case_2/serviceaccount.yaml delete mode 100644 controllers/testdata/mcad_test_results/case_3/deployment.yaml delete mode 100644 controllers/util/util.go diff --git a/Dockerfile b/Dockerfile index 302b31e1..f9614a70 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,6 @@ RUN go mod download # Copy the go source COPY main.go main.go COPY api/ api/ -COPY controllers/ controllers/ # Build USER root diff --git a/Makefile b/Makefile index e0203d27..eaf71aab 100644 --- a/Makefile +++ b/Makefile @@ -129,25 +129,10 @@ help: ## Display this help. ##@ Development -DEFAULTS_FILE := controllers/defaults.go DEFAULTS_TEST_FILE := test/support/defaults.go .PHONY: defaults defaults: - $(info Regenerating $(DEFAULTS_FILE)) - @echo "package controllers" > $(DEFAULTS_FILE) - @echo "" >> $(DEFAULTS_FILE) - @echo "// ***********************" >> $(DEFAULTS_FILE) - @echo "// DO NOT EDIT THIS FILE" >> $(DEFAULTS_FILE) - @echo "// ***********************" >> $(DEFAULTS_FILE) - @echo "" >> $(DEFAULTS_FILE) - @echo "const (" >> $(DEFAULTS_FILE) - @echo " MCADImage = \"$(MCAD_IMAGE)\"" >> $(DEFAULTS_FILE) - @echo " InstaScaleImage = \"$(INSTASCALE_IMAGE)\"" >> $(DEFAULTS_FILE) - @echo "" >> $(DEFAULTS_FILE) - @echo ")" >> $(DEFAULTS_FILE) - @echo "" >> $(DEFAULTS_FILE) - $(info Regenerating $(DEFAULTS_TEST_FILE)) @echo "package support" > $(DEFAULTS_TEST_FILE) @echo "" >> $(DEFAULTS_TEST_FILE) @@ -163,7 +148,7 @@ defaults: @echo ")" >> $(DEFAULTS_TEST_FILE) @echo "" >> $(DEFAULTS_TEST_FILE) - gofmt -w $(DEFAULTS_FILE) $(DEFAULTS_TEST_FILE) + gofmt -w $(DEFAULTS_TEST_FILE) .PHONY: manifests manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. diff --git a/controllers/config/manifest.go b/controllers/config/manifest.go deleted file mode 100644 index bfff07c9..00000000 --- a/controllers/config/manifest.go +++ /dev/null @@ -1,18 +0,0 @@ -package config - -import ( - "github.com/go-logr/logr" - mfc "github.com/manifestival/controller-runtime-client" - mf "github.com/manifestival/manifestival" - - "sigs.k8s.io/controller-runtime/pkg/client" -) - -func Manifest(cl client.Client, templatePath string, context interface{}, name string, logger logr.Logger) (mf.Manifest, error) { - m, err := mf.ManifestFrom(PathTemplateSource(templatePath, context, name), mf.UseLogger(logger)) - if err != nil { - return mf.Manifest{}, err - } - m.Client = mfc.NewClient(cl) - return m, err -} diff --git a/controllers/config/templating.go b/controllers/config/templating.go deleted file mode 100644 index cfaddf5a..00000000 --- a/controllers/config/templating.go +++ /dev/null @@ -1,48 +0,0 @@ -package config - -import ( - "bytes" - "io" - "os" - "text/template" - - mf "github.com/manifestival/manifestival" -) - -// PathPrefix is the file system path which template paths will be prefixed with. -// Default is no prefix, which causes paths to be read relative to process working dir -var PathPrefix string - -// PathTemplateSource A templating source read from a file -func PathTemplateSource(path string, context interface{}, name string) mf.Source { - f, err := os.Open(prefixedPath(path)) - if err != nil { - panic(err) - } - return templateSource(f, context, name) -} - -func prefixedPath(p string) string { - if PathPrefix != "" { - return PathPrefix + "/" + p - } - return p -} - -// A templating manifest source -func templateSource(r io.Reader, context interface{}, name string) mf.Source { - b, err := io.ReadAll(r) - if err != nil { - panic(err) - } - t, err := template.New(name).Parse(string(b)) - if err != nil { - panic(err) - } - var b2 bytes.Buffer - err = t.Execute(&b2, context) - if err != nil { - panic(err) - } - return mf.Reader(&b2) -} diff --git a/controllers/defaults.go b/controllers/defaults.go deleted file mode 100644 index 9f28b3c1..00000000 --- a/controllers/defaults.go +++ /dev/null @@ -1,10 +0,0 @@ -package controllers - -// *********************** -// DO NOT EDIT THIS FILE -// *********************** - -const ( - MCADImage = "quay.io/project-codeflare/mcad-controller:release-v1.34.1" - InstaScaleImage = "quay.io/project-codeflare/instascale-controller:v0.0.8" -) diff --git a/controllers/instascale.go b/controllers/instascale.go deleted file mode 100644 index 3a35e05a..00000000 --- a/controllers/instascale.go +++ /dev/null @@ -1,26 +0,0 @@ -package controllers - -import ( - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -var instascaleTemplates = []string{ - "instascale/configmap.yaml.tmpl", - "instascale/sa.yaml.tmpl", - "instascale/clusterrole.yaml.tmpl", - "instascale/clusterrolebinding.yaml.tmpl", - "instascale/deployment.yaml.tmpl", -} - -func (r *InstaScaleReconciler) ReconcileInstaScale(instascale *codeflarev1alpha1.InstaScale, params *InstaScaleParams) error { - - for _, template := range instascaleTemplates { - err := r.Apply(instascale, params, template) - if err != nil { - return err - } - } - - r.Log.Info("Finished applying InstaScale Resources") - return nil -} diff --git a/controllers/instascale_controller.go b/controllers/instascale_controller.go deleted file mode 100644 index 9fca52a0..00000000 --- a/controllers/instascale_controller.go +++ /dev/null @@ -1,240 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - "fmt" - "path" - - "github.com/go-logr/logr" - mf "github.com/manifestival/manifestival" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - authv1 "k8s.io/api/rbac/v1" - rbacv1 "k8s.io/api/rbac/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - "github.com/project-codeflare/codeflare-operator/controllers/config" - "github.com/project-codeflare/codeflare-operator/controllers/util" -) - -// InstaScaleReconciler reconciles a InstaScale object -type InstaScaleReconciler struct { - client.Client - Scheme *runtime.Scheme - Log logr.Logger - TemplatesPath string -} - -var instascaleClusterScopedTemplates = []string{ - "instascale/clusterrole.yaml.tmpl", - "instascale/clusterrolebinding.yaml.tmpl", -} - -func (r *InstaScaleReconciler) Apply(owner mf.Owner, params *InstaScaleParams, template string, fns ...mf.Transformer) error { - tmplManifest, err := config.Manifest(r.Client, path.Join(r.TemplatesPath, template), params, template, r.Log) - if err != nil { - return fmt.Errorf("error loading template yaml: %w", err) - } - tmplManifest, err = tmplManifest.Transform( - mf.InjectOwner(owner), - ) - if err != nil { - return err - } - - tmplManifest, err = tmplManifest.Transform(fns...) - if err != nil { - return err - } - - return tmplManifest.Apply() -} - -// TODO: Review node permissions, instascale should only require read - -// +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=instascales,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=instascales/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=instascales/finalizers,verbs=update -// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=*,resources=deployments;services,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=secrets;configmaps;nodes;services;serviceaccounts;persistentvolumes;persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=persistentvolumes;persistentvolumeclaims,verbs=* -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings,verbs=get;list;watch;create;update;delete -// +kubebuilder:rbac:groups=machine.openshift.io,resources=*,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=config.openshift.io,resources=clusterversions,verbs=get;list - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the InstaScale object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.0/pkg/reconcile -func (r *InstaScaleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - log := r.Log.WithValues("namespace", req.Namespace) - - log.V(1).Info("InstaScale reconciler called.") - - params := &InstaScaleParams{} - instascaleCustomResource := &v1alpha1.InstaScale{} - - err := r.Get(ctx, req.NamespacedName, instascaleCustomResource) - if err != nil && apierrs.IsNotFound(err) { - log.Info("Stop InstaScale reconciliation") - return ctrl.Result{}, nil - } else if err != nil { - log.Error(err, "Unable to fetch the InstaScale custom resource") - return ctrl.Result{}, err - } - - // FixMe: Hack for stubbing gvk during tests as these are not populated by test suite - // Refer to https://github.com/operator-framework/operator-sdk/issues/727#issuecomment-581169171 - // In production we expect these to be populated - if instascaleCustomResource.Kind == "" { - instascaleCustomResource = instascaleCustomResource.DeepCopy() - gvk := v1alpha1.SchemeGroupVersion.WithKind("InstaScale") - instascaleCustomResource.APIVersion, instascaleCustomResource.Kind = gvk.Version, gvk.Kind - } - - params.ExtractParams(instascaleCustomResource) - - if instascaleCustomResource.ObjectMeta.DeletionTimestamp.IsZero() { - if !controllerutil.ContainsFinalizer(instascaleCustomResource, finalizerName) { - controllerutil.AddFinalizer(instascaleCustomResource, finalizerName) - if err := r.Update(ctx, instascaleCustomResource); err != nil { - return ctrl.Result{}, err - } - } - } else { - if controllerutil.ContainsFinalizer(instascaleCustomResource, finalizerName) { - if err := r.cleanUpClusterResources(params); err != nil { - return ctrl.Result{}, err - } - controllerutil.RemoveFinalizer(instascaleCustomResource, finalizerName) - if err := r.Update(ctx, instascaleCustomResource); err != nil { - return ctrl.Result{}, err - } - } - - // Stop reconciliation as the item is being deleted - return ctrl.Result{}, nil - } - - err = r.ReconcileInstaScale(instascaleCustomResource, params) - if err != nil { - return ctrl.Result{}, err - } - - err = updateInstascaleReadyStatus(ctx, r, req, instascaleCustomResource) - if err != nil { - return ctrl.Result{}, err - } - - err = r.Client.Status().Update(ctx, instascaleCustomResource) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -func updateInstascaleReadyStatus(ctx context.Context, r *InstaScaleReconciler, req ctrl.Request, instascaleCustomResource *v1alpha1.InstaScale) error { - deployment := &appsv1.Deployment{} - err := r.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("instascale-%s", req.Name), Namespace: req.Namespace}, deployment) - if err != nil { - return err - } - r.Log.Info("Checking if deployment is ready.") - instascaleCustomResource.Status.Ready = util.IsDeploymentReady(deployment) - return nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *InstaScaleReconciler) SetupWithManager(mgr ctrl.Manager) error { - crFromLabels := handler.EnqueueRequestsFromMapFunc(func(o client.Object) []reconcile.Request { - labels := o.GetLabels() - if labels["app.kubernetes.io/managed-by"] == "InstaScale" { - crName := labels["codeflare.codeflare.dev/cr-name"] - crNamespace := labels["codeflare.codeflare.dev/cr-namespace"] - if crName != "" { - return []reconcile.Request{ - {NamespacedName: types.NamespacedName{ - Name: crName, - Namespace: crNamespace, - }}, - } - } - } - return nil - }) - return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha1.InstaScale{}). - Owns(&corev1.ConfigMap{}). - Owns(&corev1.ServiceAccount{}). - Owns(&authv1.ClusterRole{}). - Owns(&authv1.ClusterRoleBinding{}). - Owns(&appsv1.Deployment{}). - Watches( - &source.Kind{Type: &rbacv1.ClusterRole{}}, - crFromLabels, - ). - Watches( - &source.Kind{Type: &rbacv1.ClusterRoleBinding{}}, - crFromLabels, - ). - Complete(r) -} - -func (r *InstaScaleReconciler) DeleteResource(params *InstaScaleParams, template string, fns ...mf.Transformer) error { - tmplManifest, err := config.Manifest(r.Client, r.TemplatesPath+template, params, template, r.Log) - if err != nil { - return fmt.Errorf("error loading template yaml: %w", err) - } - - tmplManifest, err = tmplManifest.Transform(fns...) - if err != nil { - return err - } - - return tmplManifest.Delete() -} - -// cleanUpClusterResources will be responsible for deleting objects that do not have owner references set -func (r *InstaScaleReconciler) cleanUpClusterResources(params *InstaScaleParams) error { - for _, template := range instascaleClusterScopedTemplates { - err := r.DeleteResource(params, template) - if err != nil { - return err - } - } - return nil -} diff --git a/controllers/instascale_controller_test.go b/controllers/instascale_controller_test.go deleted file mode 100644 index 1e880190..00000000 --- a/controllers/instascale_controller_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package controllers - -import ( - "context" - - mfc "github.com/manifestival/controller-runtime-client" - mf "github.com/manifestival/manifestival" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -const ( - instascaleCRCase1 = "./testdata/instascale_test_cases/case_1.yaml" - instascaleCRCase2 = "./testdata/instascale_test_cases/case_2.yaml" - instascaleCRCase3 = "./testdata/instascale_test_cases/case_3.yaml" - instascaleConfigMap1 = "./testdata/instascale_test_results/case_1/configmap.yaml" - instascaleServiceAccount1 = "./testdata/instascale_test_results/case_1/serviceaccount.yaml" - instascaleClusterRole1 = "./testdata/instascale_test_results/case_1/clusterrole.yaml" - instascaleClusterRoleBinding1 = "./testdata/instascale_test_results/case_1/clusterrolebinding.yaml" - instascaleDeployment1 = "./testdata/instascale_test_results/case_1/deployment.yaml" - instascaleDeployment2 = "./testdata/instascale_test_results/case_2/deployment.yaml" - instascaleDeployment3 = "./testdata/instascale_test_results/case_3/deployment.yaml" -) - -func deployInstaScale(ctx context.Context, path string, opts mf.Option) { - instascale := &codeflarev1alpha1.InstaScale{} - err := convertToStructuredResource(path, instascale, opts) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient.Create(ctx, instascale)).Should(Succeed()) -} - -var _ = Describe("The Instascale Controller", func() { - client := mfc.NewClient(k8sClient) - opts := mf.UseClient(client) - ctx := context.Background() - - Context("In a namespace, when a blank InstaScale Custom Resource is deployed", func() { - - It("It should deploy InstaScale with default settings", func() { - deployInstaScale(ctx, instascaleCRCase1, opts) - compareConfigMaps(instascaleConfigMap1, opts) - compareServiceAccounts(instascaleServiceAccount1, opts) - compareDeployments(instascaleDeployment1, opts) - compareClusterRoles(instascaleClusterRole1, opts) - compareClusterRoleBindings(instascaleClusterRoleBinding1, opts) - }) - }) - - Context("In a namespace, InstaScale ControllerResources is given", func() { - - It("It should deploy InstaScale with the given ControllerResources", func() { - deployInstaScale(ctx, instascaleCRCase2, opts) - compareDeployments(instascaleDeployment2, opts) - }) - }) - - Context("When an InstaScale resource with a custom image is given", func() { - - It("It should deploy InstaScale with the given controller image", func() { - deployInstaScale(ctx, instascaleCRCase3, opts) - compareDeployments(instascaleDeployment3, opts) - }) - }) -}) diff --git a/controllers/instascale_params.go b/controllers/instascale_params.go deleted file mode 100644 index 0def6935..00000000 --- a/controllers/instascale_params.go +++ /dev/null @@ -1,61 +0,0 @@ -package controllers - -import ( - "encoding/json" - - "github.com/manifestival/manifestival" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - - instascalev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -type InstaScaleParams struct { - Name string - Namespace string - Owner manifestival.Owner - EnableMonitoring bool - MaxScaleoutAllowed int - UseMachinePools bool - ControllerResources ControllerResources - ControllerImage string -} - -type ControllerResources struct { - v1.ResourceRequirements -} - -func (c *ControllerResources) String() string { - raw, err := json.Marshal(c) - if err != nil { - return "{}" - } - return string(raw) -} - -func (p *InstaScaleParams) ExtractParams(instascale *instascalev1alpha1.InstaScale) { - p.Name = instascale.Name - p.Namespace = instascale.Namespace - p.ControllerImage = instascale.Spec.ControllerImage - if p.ControllerImage == "" { - p.ControllerImage = InstaScaleImage - } - p.Owner = instascale - p.EnableMonitoring = instascale.Spec.EnableMonitoring - p.MaxScaleoutAllowed = instascale.Spec.MaxScaleoutAllowed - p.UseMachinePools = instascale.Spec.UseMachinePools - if instascale.Spec.ControllerResources == nil { - p.ControllerResources = ControllerResources{ - v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("2G")}, - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("200M")}, - }} - } else { - p.ControllerResources = ControllerResources{*instascale.Spec.ControllerResources} - } -} diff --git a/controllers/mcad_controller.go b/controllers/mcad_controller.go deleted file mode 100644 index ce32f08b..00000000 --- a/controllers/mcad_controller.go +++ /dev/null @@ -1,247 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - mf "github.com/manifestival/manifestival" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - "github.com/project-codeflare/codeflare-operator/controllers/config" - "github.com/project-codeflare/codeflare-operator/controllers/util" -) - -const finalizerName = "codeflare.codeflare.dev/finalizer" - -// MCADReconciler reconciles a MCAD object -type MCADReconciler struct { - client.Client - Scheme *runtime.Scheme - Log logr.Logger - TemplatesPath string -} - -func (r *MCADReconciler) Apply(owner mf.Owner, params *MCADParams, template string, fns ...mf.Transformer) error { - tmplManifest, err := config.Manifest(r.Client, r.TemplatesPath+template, params, template, r.Log) - if err != nil { - return fmt.Errorf("error loading template yaml: %w", err) - } - tmplManifest, err = tmplManifest.Transform( - mf.InjectOwner(owner), - ) - if err != nil { - return err - } - - tmplManifest, err = tmplManifest.Transform(fns...) - if err != nil { - return err - } - - return tmplManifest.Apply() -} - -func (r *MCADReconciler) ApplyWithoutOwner(params *MCADParams, template string, fns ...mf.Transformer) error { - tmplManifest, err := config.Manifest(r.Client, r.TemplatesPath+template, params, template, r.Log) - if err != nil { - return fmt.Errorf("error loading template yaml: %w", err) - } - - tmplManifest, err = tmplManifest.Transform(fns...) - if err != nil { - return err - } - - return tmplManifest.Apply() -} - -func (r *MCADReconciler) DeleteResource(params *MCADParams, template string, fns ...mf.Transformer) error { - tmplManifest, err := config.Manifest(r.Client, r.TemplatesPath+template, params, template, r.Log) - if err != nil { - return fmt.Errorf("error loading template yaml: %w", err) - } - - tmplManifest, err = tmplManifest.Transform(fns...) - if err != nil { - return err - } - - return tmplManifest.Delete() -} - -// +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=codeflare.codeflare.dev,resources=mcads/finalizers,verbs=update -// +kubebuilder:rbac:groups=workload.codeflare.dev;quota.codeflare.dev,resources=queuejobs;schedulingspecs;appwrappers;appwrappers/finalizers;appwrappers/status;quotasubtrees,verbs=get;list;watch;create;update;patch;delete;deletecollection -// +kubebuilder:rbac:groups=core,resources=pods;lists;namespaces,verbs=get;list;watch;create;update;patch;delete;deletecollection -// +kubebuilder:rbac:groups=core,resources=bindings;pods/binding,verbs=create -// +kubebuilder:rbac:groups=core,resources=kube-scheduler,verbs=get;update -// +kubebuilder:rbac:groups=core,resources=endpoints;kube-scheduler,verbs=create;get;update -// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch;update -// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=patch;update -// +kubebuilder:rbac:groups=core,resources=replicationcontrollers,verbs=get;list;watch -// +kubebuilder:rbac:groups=scheduling.sigs.k8s.io,resources=podgroups,verbs=get;list;watch;create;update;patch;delete;deletecollection -// +kubebuilder:rbac:groups=apps,resources=deployments;replicasets;statefulsets,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=*,resources=deployments;services,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=secrets;configmaps;services;serviceaccounts;persistentvolumes;persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=persistentvolumes;persistentvolumeclaims,verbs=* -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings,verbs=get;list;watch;create;update;delete -// +kubebuilder:rbac:groups=custom.metrics.k8s.io,resources=*,verbs=* -// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases;kube-scheduler,verbs=create;update;get -// +kubebuilder:rbac:groups=events.k8s.io,resources=events;kube-scheduler,verbs=create;update;patch -// +kubebuilder:rbac:groups=extensions,resources=replicasets,verbs=get;list;watch -// +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch -// +kubebuilder:rbac:groups=storage.k8s.io,resources=csidrivers;csinodes;csistoragecapacities,verbs=get;list;watch -// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create -// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create - -func (r *MCADReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - log := r.Log.WithValues("namespace", req.Namespace) - - log.V(1).Info("MCAD reconciler called.") - - params := &MCADParams{} - mcadCustomResource := &v1alpha1.MCAD{} - - err := r.Get(ctx, req.NamespacedName, mcadCustomResource) - if err != nil && apierrs.IsNotFound(err) { - log.Info("Stop MCAD reconciliation") - return ctrl.Result{}, nil - } else if err != nil { - log.Error(err, "Unable to fetch the MCAD custom resource") - return ctrl.Result{}, err - } - - // FixMe: Hack for stubbing gvk during tests as these are not populated by test suite - // Refer to https://github.com/operator-framework/operator-sdk/issues/727#issuecomment-581169171 - // In production we expect these to be populated - if mcadCustomResource.Kind == "" { - mcadCustomResource = mcadCustomResource.DeepCopy() - gvk := v1alpha1.SchemeGroupVersion.WithKind("MCAD") - mcadCustomResource.APIVersion, mcadCustomResource.Kind = gvk.Version, gvk.Kind - } - - params.ExtractParams(mcadCustomResource) - - if mcadCustomResource.ObjectMeta.DeletionTimestamp.IsZero() { - if !controllerutil.ContainsFinalizer(mcadCustomResource, finalizerName) { - controllerutil.AddFinalizer(mcadCustomResource, finalizerName) - if err := r.Update(ctx, mcadCustomResource); err != nil { - return ctrl.Result{}, err - } - } - } else { - if controllerutil.ContainsFinalizer(mcadCustomResource, finalizerName) { - if err := r.cleanUpOwnerLessResources(params); err != nil { - return ctrl.Result{}, err - } - controllerutil.RemoveFinalizer(mcadCustomResource, finalizerName) - if err := r.Update(ctx, mcadCustomResource); err != nil { - return ctrl.Result{}, err - } - } - - // Stop reconciliation as the item is being deleted - return ctrl.Result{}, nil - } - - log.V(1).Info("ReconcileMCAD called.") - err = r.ReconcileMCAD(mcadCustomResource, params) - if err != nil { - return ctrl.Result{}, err - } - - err = updateMCADReadyStatus(ctx, r, req, mcadCustomResource) - if err != nil { - return ctrl.Result{}, err - } - err = r.Client.Status().Update(ctx, mcadCustomResource) - if err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil -} - -func updateMCADReadyStatus(ctx context.Context, r *MCADReconciler, req ctrl.Request, mcadCustomResource *v1alpha1.MCAD) error { - deployment := &appsv1.Deployment{} - err := r.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("mcad-controller-%s", req.Name), Namespace: req.Namespace}, deployment) - if err != nil { - return err - } - r.Log.Info("Checking if MCAD deployment is ready.") - mcadCustomResource.Status.Ready = util.IsDeploymentReady(deployment) - return nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *MCADReconciler) SetupWithManager(mgr ctrl.Manager) error { - crFromLabels := handler.EnqueueRequestsFromMapFunc(func(o client.Object) []reconcile.Request { - labels := o.GetLabels() - if labels["app.kubernetes.io/managed-by"] == "MCAD" { - crName := labels["codeflare.codeflare.dev/cr-name"] - crNamespace := labels["codeflare.codeflare.dev/cr-namespace"] - if crName != "" { - return []reconcile.Request{ - {NamespacedName: types.NamespacedName{ - Name: crName, - Namespace: crNamespace, - }}, - } - } - } - return nil - }) - return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha1.MCAD{}). - Owns(&corev1.ConfigMap{}). - Owns(&corev1.Service{}). - Owns(&corev1.ServiceAccount{}). - Owns(&rbacv1.RoleBinding{}). - Owns(&appsv1.Deployment{}). - Watches( - &source.Kind{Type: &rbacv1.ClusterRole{}}, - crFromLabels, - ). - Watches( - &source.Kind{Type: &rbacv1.ClusterRoleBinding{}}, - crFromLabels, - ). - Complete(r) -} - -// cleanUpClusterResources will be responsible for deleting objects that do not have owner references set -func (r *MCADReconciler) cleanUpOwnerLessResources(params *MCADParams) error { - return r.deleteOwnerLessObjects(params) -} diff --git a/controllers/mcad_controller_test.go b/controllers/mcad_controller_test.go deleted file mode 100644 index 7ffd30cc..00000000 --- a/controllers/mcad_controller_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package controllers - -import ( - "context" - - mfc "github.com/manifestival/controller-runtime-client" - mf "github.com/manifestival/manifestival" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -const ( - mcadCRCase1 = "./testdata/mcad_test_cases/case_1.yaml" - mcadConfigMap1 = "./testdata/mcad_test_results/case_1/configmap.yaml" - mcadService1 = "./testdata/mcad_test_results/case_1/service.yaml" - mcadServiceAccount1 = "./testdata/mcad_test_results/case_1/serviceaccount.yaml" - mcadCRCase2 = "./testdata/mcad_test_cases/case_2.yaml" - mcadConfigMap2 = "./testdata/mcad_test_results/case_2/configmap.yaml" - mcadService2 = "./testdata/mcad_test_results/case_2/service.yaml" - mcadServiceAccount2 = "./testdata/mcad_test_results/case_2/serviceaccount.yaml" - mcadCRCase3 = "./testdata/mcad_test_cases/case_3.yaml" - mcadDeployment3 = "./testdata/mcad_test_results/case_3/deployment.yaml" -) - -func deployMCAD(ctx context.Context, path string, opts mf.Option) { - mcad := &codeflarev1alpha1.MCAD{} - err := convertToStructuredResource(path, mcad, opts) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient.Create(ctx, mcad)).Should(Succeed()) -} - -var _ = Describe("The MCAD Controller", func() { - client := mfc.NewClient(k8sClient) - opts := mf.UseClient(client) - ctx := context.Background() - - Context("In a namespace, when a blank MCAD Custom Resource is deployed", func() { - - It("It should create a configmap", func() { - deployMCAD(ctx, mcadCRCase1, opts) - compareConfigMaps(mcadConfigMap1, opts) - compareServiceAccounts(mcadServiceAccount1, opts) - compareServices(mcadService1, opts) - }) - }) - - Context("In a namespace, when a populated MCAD Custom Resource is deployed", func() { - - It("It should create a configmap", func() { - deployMCAD(ctx, mcadCRCase2, opts) - compareConfigMaps(mcadConfigMap2, opts) - compareServiceAccounts(mcadServiceAccount2, opts) - compareServices(mcadService2, opts) - }) - }) - - Context("In a namespace, when a MCAD resource with a custom image is deployed", func() { - - It("It should create a deployment", func() { - deployMCAD(ctx, mcadCRCase3, opts) - compareDeployments(mcadDeployment3, opts) - }) - }) -}) diff --git a/controllers/mcad_params.go b/controllers/mcad_params.go deleted file mode 100644 index 846e4c73..00000000 --- a/controllers/mcad_params.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - mf "github.com/manifestival/manifestival" - - mcadv1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -type MCADParams struct { - Name string - Namespace string - Owner mf.Owner - EnableMonitoring bool - MultiCluster bool - DispatcherMode bool - PreemptionEnabled bool - AgentConfigs string - QuotaRestURL string - PodCreationTimeout int - ControllerResources ControllerResources - ControllerImage string -} - -// ExtractParams is currently a straight-up copy. We can add in more complex validation at a later date -func (p *MCADParams) ExtractParams(mcad *mcadv1alpha1.MCAD) { - p.Name = mcad.Name - p.Namespace = mcad.Namespace - p.ControllerImage = mcad.Spec.ControllerImage - if p.ControllerImage == "" { - p.ControllerImage = MCADImage - } - p.Owner = mcad - p.EnableMonitoring = mcad.Spec.EnableMonitoring - p.MultiCluster = mcad.Spec.MultiCluster - p.DispatcherMode = mcad.Spec.DispatcherMode - p.PreemptionEnabled = mcad.Spec.PreemptionEnabled - p.AgentConfigs = mcad.Spec.AgentConfigs - p.QuotaRestURL = mcad.Spec.QuotaRestURL - p.PodCreationTimeout = mcad.Spec.PodCreationTimeout - p.ControllerResources = ControllerResources{mcad.Spec.ControllerResources} -} diff --git a/controllers/multi_cluster_app_dispatcher.go b/controllers/multi_cluster_app_dispatcher.go deleted file mode 100644 index 3d80e3c5..00000000 --- a/controllers/multi_cluster_app_dispatcher.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -var multiClusterAppDispatcherTemplates = []string{ - "mcad/configmap.yaml.tmpl", - "mcad/serviceaccount.yaml.tmpl", - "mcad/deployment.yaml.tmpl", -} -var ownerLessmultiClusterAppDispatcherTemplates = []string{ - "mcad/clusterrole_mcad-controller.yaml.tmpl", - "mcad/clusterrolebinding_mcad-controller.yaml.tmpl", - "mcad/clusterrolebinding_mcad-controller-kube-scheduler.yaml.tmpl", - "mcad/clusterrolebinding_mcad-edit.yaml.tmpl", -} - -func (r *MCADReconciler) ReconcileMCAD(mcad *codeflarev1alpha1.MCAD, params *MCADParams) error { - - for _, template := range multiClusterAppDispatcherTemplates { - r.Log.Info("Applying " + template) - err := r.Apply(mcad, params, template) - if err != nil { - return err - } - } - - for _, template := range ownerLessmultiClusterAppDispatcherTemplates { - r.Log.Info("Applying " + template) - err := r.ApplyWithoutOwner(params, template) - if err != nil { - return err - } - } - - r.Log.Info("Finished applying MultiClusterAppDispatcher Resources") - return nil -} - -func (r *MCADReconciler) deleteOwnerLessObjects(params *MCADParams) error { - for _, template := range ownerLessmultiClusterAppDispatcherTemplates { - r.Log.Info("Deleting Ownerless object: " + template) - err := r.DeleteResource(params, template) - if err != nil { - return err - } - } - return nil -} diff --git a/controllers/suite_test.go b/controllers/suite_test.go deleted file mode 100644 index 5ee347ba..00000000 --- a/controllers/suite_test.go +++ /dev/null @@ -1,262 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - "path/filepath" - "testing" - "time" - - mf "github.com/manifestival/manifestival" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "go.uber.org/zap/zapcore" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/types" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/kubernetes/scheme" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - - "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - "github.com/project-codeflare/codeflare-operator/controllers/util" - // +kubebuilder:scaffold:imports -) - -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - -var ( - cfg *rest.Config - k8sClient client.Client - testEnv *envtest.Environment - ctx context.Context - cancel context.CancelFunc -) - -const ( - workingNamespace = "default" - timeout = time.Second * 30 - interval = time.Millisecond * 10 -) - -func TestAPIs(t *testing.T) { - RegisterFailHandler(Fail) - - RunSpecs(t, "Controllers Suite") -} - -var _ = BeforeSuite(func() { - ctx, cancel = context.WithCancel(context.TODO()) - - // Initialize logger - opts := zap.Options{ - Development: true, - TimeEncoder: zapcore.TimeEncoderOfLayout(time.RFC3339), - } - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseFlagOptions(&opts))) - - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - } - - var err error - // cfg is defined in this file globally. - cfg, err = testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - // Register API objects - utilruntime.Must(clientgoscheme.AddToScheme(scheme.Scheme)) - utilruntime.Must(v1alpha1.AddToScheme(scheme.Scheme)) - // +kubebuilder:scaffold:scheme - - // Initialize Kubernetes client - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) - - // Setup controller manager - mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, - LeaderElection: false, - MetricsBindAddress: "0", - }) - Expect(err).NotTo(HaveOccurred()) - - err = (&MCADReconciler{ - Client: k8sClient, - Log: ctrl.Log.WithName("controllers").WithName("mcad-controller"), - Scheme: scheme.Scheme, - TemplatesPath: "../config/internal/", - }).SetupWithManager(mgr) - Expect(err).ToNot(HaveOccurred()) - - err = (&InstaScaleReconciler{ - Client: k8sClient, - Log: ctrl.Log.WithName("controllers").WithName("instascale-controller"), - Scheme: scheme.Scheme, - TemplatesPath: "../config/internal/", - }).SetupWithManager(mgr) - Expect(err).ToNot(HaveOccurred()) - - // Start the manager - go func() { - defer GinkgoRecover() - err = mgr.Start(ctx) - Expect(err).ToNot(HaveOccurred(), "Failed to run manager") - }() - -}) - -var _ = AfterSuite(func() { - // Give some time to allow workers to gracefully shutdown - time.Sleep(5 * time.Second) - cancel() - By("tearing down the test environment") - time.Sleep(1 * time.Second) - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) -}) - -// Cleanup resources to not contaminate between tests -var _ = AfterEach(func() { - inNamespace := client.InNamespace(workingNamespace) - Expect(k8sClient.DeleteAllOf(context.TODO(), &v1alpha1.MCAD{}, inNamespace)).ToNot(HaveOccurred()) - Expect(k8sClient.DeleteAllOf(context.TODO(), &v1alpha1.InstaScale{}, inNamespace)).ToNot(HaveOccurred()) - -}) - -func convertToStructuredResource(path string, out interface{}, opts mf.Option) error { - m, err := mf.ManifestFrom(mf.Recursive(path), opts) - if err != nil { - return err - } - m, err = m.Transform(mf.InjectNamespace(workingNamespace)) - if err != nil { - return err - } - err = scheme.Scheme.Convert(&m.Resources()[0], out, nil) - if err != nil { - return err - } - return nil -} - -func compareConfigMaps(path string, opts mf.Option) { - expectedConfigMap := &corev1.ConfigMap{} - Expect(convertToStructuredResource(path, expectedConfigMap, opts)).NotTo(HaveOccurred()) - - actualConfigMap := &corev1.ConfigMap{} - Eventually(func() error { - namespacedNamed := types.NamespacedName{Name: expectedConfigMap.Name, Namespace: workingNamespace} - return k8sClient.Get(ctx, namespacedNamed, actualConfigMap) - }, timeout, interval).ShouldNot(HaveOccurred()) - - Expect(util.ConfigMapsAreEqual(*expectedConfigMap, *actualConfigMap)).Should(BeTrue()) -} - -// func compareRoleBindings(path string, opts mf.Option) { -// expectedRB := &k8srbacv1.RoleBinding{} -// Expect(convertToStructuredResource(path, expectedRB, opts)).NotTo(HaveOccurred()) -// expectedRB.Subjects[0].Namespace = workingNamespace -// -// actualRB := &k8srbacv1.RoleBinding{} -// Eventually(func() error { -// namespacedNamed := types.NamespacedName{Name: expectedRB.Name, Namespace: workingNamespace} -// return k8sClient.Get(ctx, namespacedNamed, actualRB) -// }, timeout, interval).ShouldNot(HaveOccurred()) -// -// Expect(util.RoleBindingsAreEqual(*expectedRB, *actualRB)).Should(BeTrue()) -// } - -func compareServiceAccounts(path string, opts mf.Option) { - expectedSA := &corev1.ServiceAccount{} - Expect(convertToStructuredResource(path, expectedSA, opts)).NotTo(HaveOccurred()) - expectedSA.Namespace = workingNamespace - - actualSA := &corev1.ServiceAccount{} - Eventually(func() error { - namespacedNamed := types.NamespacedName{Name: expectedSA.Name, Namespace: workingNamespace} - return k8sClient.Get(ctx, namespacedNamed, actualSA) - }, timeout, interval).ShouldNot(HaveOccurred()) - - Expect(util.ServiceAccountsAreEqual(*expectedSA, *actualSA)).Should(BeTrue()) -} - -func compareServices(path string, opts mf.Option) { - expectedService := &corev1.Service{} - Expect(convertToStructuredResource(path, expectedService, opts)).NotTo(HaveOccurred()) - - actualService := &corev1.Service{} - Eventually(func() error { - namespacedNamed := types.NamespacedName{Name: expectedService.Name, Namespace: workingNamespace} - return k8sClient.Get(ctx, namespacedNamed, actualService) - }, timeout, interval).ShouldNot(HaveOccurred()) - - Expect(util.ServicesAreEqual(*expectedService, *actualService)).Should(BeTrue()) -} - -func compareDeployments(path string, opts mf.Option) { - expectedDeployment := &appsv1.Deployment{} - Expect(convertToStructuredResource(path, expectedDeployment, opts)).NotTo(HaveOccurred()) - - actualDeployment := &appsv1.Deployment{} - Eventually(func() error { - namespacedNamed := types.NamespacedName{Name: expectedDeployment.Name, Namespace: workingNamespace} - return k8sClient.Get(ctx, namespacedNamed, actualDeployment) - }, timeout, interval).ShouldNot(HaveOccurred()) - - Expect(util.DeploymentsAreEqual(*expectedDeployment, *actualDeployment)).Should(BeTrue()) -} - -func compareClusterRoles(path string, opts mf.Option) { - expectedClusterRole := &rbacv1.ClusterRole{} - Expect(convertToStructuredResource(path, expectedClusterRole, opts)).NotTo(HaveOccurred()) - - actualClusterRole := &rbacv1.ClusterRole{} - Eventually(func() error { - namespacedNamed := types.NamespacedName{Name: expectedClusterRole.Name, Namespace: workingNamespace} - return k8sClient.Get(ctx, namespacedNamed, actualClusterRole) - }, timeout, interval).ShouldNot(HaveOccurred()) - - Expect(util.ClusterRolesAreEqual(*expectedClusterRole, *actualClusterRole)).Should(BeTrue()) -} - -func compareClusterRoleBindings(path string, opts mf.Option) { - expectedClusterRoleBinding := &rbacv1.ClusterRoleBinding{} - Expect(convertToStructuredResource(path, expectedClusterRoleBinding, opts)).NotTo(HaveOccurred()) - - actualClusterRoleBinding := &rbacv1.ClusterRoleBinding{} - Eventually(func() error { - namespacedNamed := types.NamespacedName{Name: expectedClusterRoleBinding.Name, Namespace: workingNamespace} - return k8sClient.Get(ctx, namespacedNamed, actualClusterRoleBinding) - }, timeout, interval).ShouldNot(HaveOccurred()) - - Expect(util.ClusterRoleBindingsAreEqual(*expectedClusterRoleBinding, *actualClusterRoleBinding)).Should(BeTrue()) -} diff --git a/controllers/testdata/instascale_test_cases/case_1.yaml b/controllers/testdata/instascale_test_cases/case_1.yaml deleted file mode 100644 index b63d46a8..00000000 --- a/controllers/testdata/instascale_test_cases/case_1.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: codeflare.codeflare.dev/v1alpha1 -kind: InstaScale -metadata: - name: example -spec: {} diff --git a/controllers/testdata/instascale_test_cases/case_2.yaml b/controllers/testdata/instascale_test_cases/case_2.yaml deleted file mode 100644 index 1bfaaaca..00000000 --- a/controllers/testdata/instascale_test_cases/case_2.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: codeflare.codeflare.dev/v1alpha1 -kind: InstaScale -metadata: - name: example2 -spec: - controllerResources: - limits: - cpu: '1' - memory: 1G - requests: - cpu: '1' - memory: 1G diff --git a/controllers/testdata/instascale_test_cases/case_3.yaml b/controllers/testdata/instascale_test_cases/case_3.yaml deleted file mode 100644 index a7d807f0..00000000 --- a/controllers/testdata/instascale_test_cases/case_3.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: codeflare.codeflare.dev/v1alpha1 -kind: InstaScale -metadata: - name: example-custom-image -spec: - controllerImage: quay.io/project-codeflare/instascale-controller:custom diff --git a/controllers/testdata/instascale_test_results/case_1/clusterrole.yaml b/controllers/testdata/instascale_test_results/case_1/clusterrole.yaml deleted file mode 100644 index 150863ed..00000000 --- a/controllers/testdata/instascale_test_results/case_1/clusterrole.yaml +++ /dev/null @@ -1,68 +0,0 @@ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: instascale-example-cr -rules: - - verbs: - - list - - watch - - get - - create - - update - - delete - - patch - apiGroups: - - '' - resources: - - nodes - - configmaps - - - verbs: - - get - apiGroups: - - '' - resourceNames: - - instascale-ocm-secret - resources: - - secrets - - - verbs: - - get - - list - apiGroups: - - config.openshift.io - resources: - - clusterversions - - - verbs: - - list - - watch - - get - apiGroups: - - apps - resources: - - deployments - - verbs: - - list - - watch - - get - - create - - update - - delete - - patch - apiGroups: - - machine.openshift.io - resources: - - '*' - - verbs: - - list - - watch - - get - - create - - update - - delete - - patch - apiGroups: - - workload.codeflare.dev - resources: - - appwrappers diff --git a/controllers/testdata/instascale_test_results/case_1/clusterrolebinding.yaml b/controllers/testdata/instascale_test_results/case_1/clusterrolebinding.yaml deleted file mode 100644 index 5b0f84d9..00000000 --- a/controllers/testdata/instascale_test_results/case_1/clusterrolebinding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: instascale-example-crb -subjects: - - kind: ServiceAccount - name: instascale-example-sa - namespace: default -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: instascale-example-cr diff --git a/controllers/testdata/instascale_test_results/case_1/configmap.yaml b/controllers/testdata/instascale_test_results/case_1/configmap.yaml deleted file mode 100644 index a64050bd..00000000 --- a/controllers/testdata/instascale_test_results/case_1/configmap.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: instascale-config - namespace: default - labels: - app: instascale-example - component: instascale -data: - maxScaleoutAllowed: '15' - useMachinePools: 'false' diff --git a/controllers/testdata/instascale_test_results/case_1/deployment.yaml b/controllers/testdata/instascale_test_results/case_1/deployment.yaml deleted file mode 100644 index 4587ba19..00000000 --- a/controllers/testdata/instascale_test_results/case_1/deployment.yaml +++ /dev/null @@ -1,30 +0,0 @@ -kind: Deployment -apiVersion: apps/v1 -metadata: - name: instascale-example - namespace: codeflare-operator-system - labels: - app: instascale-example -spec: - replicas: 1 - selector: - matchLabels: - app: instascale-example - template: - metadata: - labels: - app: instascale-example - spec: - containers: - - name: instascale - args: - - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.8 - resources: - limits: - cpu: '2' - memory: 2G - requests: - cpu: '200m' - memory: 200M - serviceAccountName: instascale-example-sa diff --git a/controllers/testdata/instascale_test_results/case_1/serviceaccount.yaml b/controllers/testdata/instascale_test_results/case_1/serviceaccount.yaml deleted file mode 100644 index 4a7724e6..00000000 --- a/controllers/testdata/instascale_test_results/case_1/serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -kind: ServiceAccount -apiVersion: v1 -metadata: - name: instascale-example-sa - namespace: default diff --git a/controllers/testdata/instascale_test_results/case_2/deployment.yaml b/controllers/testdata/instascale_test_results/case_2/deployment.yaml deleted file mode 100644 index f5b353ff..00000000 --- a/controllers/testdata/instascale_test_results/case_2/deployment.yaml +++ /dev/null @@ -1,30 +0,0 @@ -kind: Deployment -apiVersion: apps/v1 -metadata: - name: instascale-example2 - namespace: codeflare-operator-system - labels: - app: instascale-example2 -spec: - replicas: 1 - selector: - matchLabels: - app: instascale-example2 - template: - metadata: - labels: - app: instascale-example2 - spec: - containers: - - name: instascale - args: - - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:v0.0.8 - resources: - limits: - cpu: '1' - memory: 1G - requests: - cpu: '1' - memory: 1G - serviceAccountName: instascale-example2-sa diff --git a/controllers/testdata/instascale_test_results/case_3/deployment.yaml b/controllers/testdata/instascale_test_results/case_3/deployment.yaml deleted file mode 100644 index 0e608110..00000000 --- a/controllers/testdata/instascale_test_results/case_3/deployment.yaml +++ /dev/null @@ -1,30 +0,0 @@ -kind: Deployment -apiVersion: apps/v1 -metadata: - name: instascale-example-custom-image - namespace: codeflare-operator-system - labels: - app: instascale-example-custom-image -spec: - replicas: 1 - selector: - matchLabels: - app: instascale-example-custom-image - template: - metadata: - labels: - app: instascale-example-custom-image - spec: - containers: - - name: instascale - args: - - "--configs-namespace=default" - image: quay.io/project-codeflare/instascale-controller:custom - resources: - limits: - cpu: '2' - memory: 2G - requests: - cpu: '200m' - memory: 200M - serviceAccountName: instascale-example-custom-image-sa diff --git a/controllers/testdata/mcad_test_cases/case_1.yaml b/controllers/testdata/mcad_test_cases/case_1.yaml deleted file mode 100644 index ac338a4c..00000000 --- a/controllers/testdata/mcad_test_cases/case_1.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: codeflare.codeflare.dev/v1alpha1 -kind: MCAD -metadata: - name: blank-custom-resource -spec: {} diff --git a/controllers/testdata/mcad_test_cases/case_2.yaml b/controllers/testdata/mcad_test_cases/case_2.yaml deleted file mode 100644 index 75b5037e..00000000 --- a/controllers/testdata/mcad_test_cases/case_2.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: codeflare.codeflare.dev/v1alpha1 -kind: MCAD -metadata: - name: populated-custom-resource -spec: - podCreationTimeout: 300 - quotaRestURL: 'bar.com' - agentConfigs: 'foo' - controllerResources: - limits: - cpu: '2' - memory: 2G - requests: - cpu: '2' - memory: 2G diff --git a/controllers/testdata/mcad_test_cases/case_3.yaml b/controllers/testdata/mcad_test_cases/case_3.yaml deleted file mode 100644 index c048e2e8..00000000 --- a/controllers/testdata/mcad_test_cases/case_3.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: codeflare.codeflare.dev/v1alpha1 -kind: MCAD -metadata: - name: custom-image -spec: - controllerImage: quay.io/project-codeflare/mcad-controller:custom diff --git a/controllers/testdata/mcad_test_results/case_1/configmap.yaml b/controllers/testdata/mcad_test_results/case_1/configmap.yaml deleted file mode 100644 index d14dbd0a..00000000 --- a/controllers/testdata/mcad_test_results/case_1/configmap.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: mcad-blank-custom-resource-config - namespace: default - labels: - app: mcad-blank-custom-resource - component: multi-cluster-app-dispatcher -data: - DISPATCHER_MODE: 'false' - PREEMPTION: 'false' diff --git a/controllers/testdata/mcad_test_results/case_1/serviceaccount.yaml b/controllers/testdata/mcad_test_results/case_1/serviceaccount.yaml deleted file mode 100644 index 56786f25..00000000 --- a/controllers/testdata/mcad_test_results/case_1/serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -kind: ServiceAccount -apiVersion: v1 -metadata: - name: mcad-controller-blank-custom-resource - namespace: default diff --git a/controllers/testdata/mcad_test_results/case_2/configmap.yaml b/controllers/testdata/mcad_test_results/case_2/configmap.yaml deleted file mode 100644 index e8aa40a9..00000000 --- a/controllers/testdata/mcad_test_results/case_2/configmap.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: mcad-populated-custom-resource-config - namespace: default - labels: - app: mcad-populated-custom-resource - component: multi-cluster-app-dispatcher -data: - DISPATCHER_MODE: 'false' - PREEMPTION: 'false' - DISPATCHER_AGENT_CONFIGS: 'foo' - DISPATCH_RESOURCE_RESERVATION_TIMEOUT: '300' - QUOTA_REST_URL: 'bar.com' diff --git a/controllers/testdata/mcad_test_results/case_2/serviceaccount.yaml b/controllers/testdata/mcad_test_results/case_2/serviceaccount.yaml deleted file mode 100644 index f2003528..00000000 --- a/controllers/testdata/mcad_test_results/case_2/serviceaccount.yaml +++ /dev/null @@ -1,5 +0,0 @@ -kind: ServiceAccount -apiVersion: v1 -metadata: - name: mcad-controller-populated-custom-resource - namespace: default diff --git a/controllers/testdata/mcad_test_results/case_3/deployment.yaml b/controllers/testdata/mcad_test_results/case_3/deployment.yaml deleted file mode 100644 index 03f8dffd..00000000 --- a/controllers/testdata/mcad_test_results/case_3/deployment.yaml +++ /dev/null @@ -1,38 +0,0 @@ -kind: Deployment -apiVersion: apps/v1 -metadata: - name: mcad-controller-custom-image - namespace: default - labels: - app: mcad-custom-image - component: multi-cluster-application-dispatcher -spec: - replicas: 1 - selector: - matchLabels: - app: mcad-custom-image - template: - metadata: - labels: - app: mcad-custom-image - component: multi-cluster-application-dispatcher - spec: - containers: - - name: mcad-controller - args: ["--v", "4", "--logtostderr"] - command: - - mcad-controller - envFrom: - - configMapRef: - name: mcad-custom-image-config - image: quay.io/project-codeflare/mcad-controller:custom - imagePullPolicy: Always - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - name: temp-vol - mountPath: /tmp - serviceAccountName: mcad-controller-custom-image - volumes: - - name: temp-vol - emptyDir: {} diff --git a/controllers/util/util.go b/controllers/util/util.go deleted file mode 100644 index 00edb3ab..00000000 --- a/controllers/util/util.go +++ /dev/null @@ -1,183 +0,0 @@ -package util - -import ( - "fmt" - "reflect" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" -) - -func notEqualMsg(value string) { - print(fmt.Sprintf("%s are not equal.", value)) -} - -func ConfigMapsAreEqual(expected corev1.ConfigMap, actual corev1.ConfigMap) bool { - if expected.Name != actual.Name { - notEqualMsg("Configmap Names") - return false - } - - if !reflect.DeepEqual(expected.Data, actual.Data) { - notEqualMsg("Configmap Data values") - return false - } - return true -} - -//func RoleBindingsAreEqual(rb1 k8srbacv1.RoleBinding, rb2 k8srbacv1.RoleBinding) bool { -// -// if !reflect.DeepEqual(rb1.ObjectMeta.Labels, rb2.ObjectMeta.Labels) { -// notEqualMsg("Rolebinding labels") -// return false -// } -// if !reflect.DeepEqual(rb1.Subjects, rb2.Subjects) { -// notEqualMsg("Rolebinding subjects") -// return false -// } -// if !reflect.DeepEqual(rb1.RoleRef, rb2.RoleRef) { -// notEqualMsg("Rolebinding role references") -// return false -// } -// return true -//} - -func ServiceAccountsAreEqual(sa1 corev1.ServiceAccount, sa2 corev1.ServiceAccount) bool { - if !reflect.DeepEqual(sa1.ObjectMeta.Labels, sa2.ObjectMeta.Labels) { - notEqualMsg("ServiceAccount labels") - return false - } - if !reflect.DeepEqual(sa1.Name, sa2.Name) { - notEqualMsg("ServiceAccount names") - return false - } - if !reflect.DeepEqual(sa1.Namespace, sa2.Namespace) { - notEqualMsg("ServiceAccount namespaces") - return false - } - return true -} - -func ServicesAreEqual(service1 corev1.Service, service2 corev1.Service) bool { - if !reflect.DeepEqual(service1.ObjectMeta.Labels, service2.ObjectMeta.Labels) { - notEqualMsg("Service labels") - return false - } - if !reflect.DeepEqual(service1.Name, service2.Name) { - notEqualMsg("Service Names") - return false - } - if !reflect.DeepEqual(service1.Namespace, service2.Namespace) { - notEqualMsg("Service namespaces") - return false - } - if !reflect.DeepEqual(service1.Spec.Selector, service2.Spec.Selector) { - notEqualMsg("Service Selectors") - return false - } - if !reflect.DeepEqual(service1.Spec.Ports, service2.Spec.Ports) { - notEqualMsg("Service Ports") - return false - } - return true -} - -func DeploymentsAreEqual(dp1 appsv1.Deployment, dp2 appsv1.Deployment) bool { - - if !reflect.DeepEqual(dp1.ObjectMeta.Labels, dp2.ObjectMeta.Labels) { - notEqualMsg("labels") - return false - } - - if !reflect.DeepEqual(dp1.Spec.Selector, dp2.Spec.Selector) { - notEqualMsg("selector") - return false - } - - if !reflect.DeepEqual(dp1.Spec.Template.ObjectMeta, dp2.Spec.Template.ObjectMeta) { - notEqualMsg("Object MetaData") - return false - } - - if !reflect.DeepEqual(dp1.Spec.Template.Spec.Volumes, dp2.Spec.Template.Spec.Volumes) { - notEqualMsg("Volumes") - return false - } - - if len(dp1.Spec.Template.Spec.Containers) != len(dp2.Spec.Template.Spec.Containers) { - notEqualMsg("Containers") - return false - } - for i := range dp1.Spec.Template.Spec.Containers { - c1 := dp1.Spec.Template.Spec.Containers[i] - c2 := dp2.Spec.Template.Spec.Containers[i] - if !reflect.DeepEqual(c1.Env, c2.Env) { - notEqualMsg("Container Env") - return false - } - if !reflect.DeepEqual(c1.Ports, c2.Ports) { - notEqualMsg("Container Ports") - return false - } - if !reflect.DeepEqual(c1.Resources, c2.Resources) { - notEqualMsg("Container Resources") - return false - } - if !reflect.DeepEqual(c1.VolumeMounts, c2.VolumeMounts) { - notEqualMsg("Container VolumeMounts") - return false - } - if !reflect.DeepEqual(c1.Args, c2.Args) { - notEqualMsg("Container Args") - return false - } - if c1.Name != c2.Name { - notEqualMsg("Container Name") - return false - } - if c1.Image != c2.Image { - notEqualMsg("Container Image") - return false - } - } - - return true -} - -func ClusterRolesAreEqual(cr1 rbacv1.ClusterRole, cr2 rbacv1.ClusterRole) bool { - if !reflect.DeepEqual(cr1.Rules, cr2.Rules) { - notEqualMsg("rules") - return false - } - if !reflect.DeepEqual(cr1.ObjectMeta.Name, cr2.ObjectMeta.Name) { - notEqualMsg("name") - return false - } - return true -} - -func ClusterRoleBindingsAreEqual(crb1 rbacv1.ClusterRoleBinding, crb2 rbacv1.ClusterRoleBinding) bool { - if !reflect.DeepEqual(crb1.Subjects, crb2.Subjects) { - notEqualMsg("subjects") - return false - } - if !reflect.DeepEqual(crb1.RoleRef, crb2.RoleRef) { - notEqualMsg("roleRef") - return false - } - if !reflect.DeepEqual(crb1.ObjectMeta.Name, crb2.ObjectMeta.Name) { - notEqualMsg("name") - return false - } - return true -} - -func IsDeploymentReady(deployment *appsv1.Deployment) bool { - for _, condition := range deployment.Status.Conditions { - if condition.Type == appsv1.DeploymentAvailable && condition.Status == corev1.ConditionTrue { - return true - } - } - return false -} diff --git a/main.go b/main.go index 8a8111c6..73288133 100644 --- a/main.go +++ b/main.go @@ -119,26 +119,6 @@ func main() { exitOnError(instascaleController.SetupWithManager(mgr), "Error setting up InstaScale controller") - // if err = (&controllers.MCADReconciler{ - // Client: mgr.GetClient(), - // Scheme: mgr.GetScheme(), - // Log: ctrl.Log, - // TemplatesPath: templatesPath, - // }).SetupWithManager(mgr); err != nil { - // setupLog.Error(err, "unable to create controller", "controller", "MCAD") - // os.Exit(1) - // } - // if err = (&controllers.InstaScaleReconciler{ - // Client: mgr.GetClient(), - // Scheme: mgr.GetScheme(), - // Log: ctrl.Log, - // TemplatesPath: templatesPath, - // }).SetupWithManager(mgr); err != nil { - // setupLog.Error(err, "unable to create controller", "controller", "InstaScale") - // os.Exit(1) - // } - // +kubebuilder:scaffold:builder - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) From b559be5da5517f26be21a13c738fd33f22699d34 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 11 Aug 2023 18:05:52 +0200 Subject: [PATCH 120/377] Remove client package --- .github/workflows/verify_generated_files.yml | 11 - Makefile | 63 ----- .../codeflare/v1alpha1/instascale.go | 219 --------------- .../codeflare/v1alpha1/instascalespec.go | 79 ------ .../codeflare/v1alpha1/instascalestatus.go | 39 --- .../codeflare/v1alpha1/mcad.go | 219 --------------- .../codeflare/v1alpha1/mcadspec.go | 115 -------- .../codeflare/v1alpha1/mcadstatus.go | 39 --- .../applyconfiguration/internal/internal.go | 62 ----- client/applyconfiguration/utils.go | 48 ---- client/clientset/versioned/clientset.go | 121 --------- .../versioned/fake/clientset_generated.go | 86 ------ client/clientset/versioned/fake/doc.go | 20 -- client/clientset/versioned/fake/register.go | 57 ---- client/clientset/versioned/scheme/doc.go | 20 -- client/clientset/versioned/scheme/register.go | 57 ---- .../codeflare/v1alpha1/codeflare_client.go | 113 -------- .../versioned/typed/codeflare/v1alpha1/doc.go | 20 -- .../typed/codeflare/v1alpha1/fake/doc.go | 20 -- .../v1alpha1/fake/fake_codeflare_client.go | 45 --- .../v1alpha1/fake/fake_instascale.go | 190 ------------- .../codeflare/v1alpha1/fake/fake_mcad.go | 190 ------------- .../codeflare/v1alpha1/generated_expansion.go | 23 -- .../typed/codeflare/v1alpha1/instascale.go | 257 ------------------ .../typed/codeflare/v1alpha1/mcad.go | 257 ------------------ .../externalversions/codeflare/interface.go | 46 ---- .../codeflare/v1alpha1/instascale.go | 91 ------- .../codeflare/v1alpha1/interface.go | 52 ---- .../codeflare/v1alpha1/mcad.go | 91 ------- client/informer/externalversions/factory.go | 252 ----------------- client/informer/externalversions/generic.go | 65 ----- .../internalinterfaces/factory_interfaces.go | 41 --- .../codeflare/v1alpha1/expansion_generated.go | 35 --- .../listers/codeflare/v1alpha1/instascale.go | 100 ------- client/listers/codeflare/v1alpha1/mcad.go | 100 ------- test/support/client.go | 39 +-- 36 files changed, 12 insertions(+), 3270 deletions(-) delete mode 100644 client/applyconfiguration/codeflare/v1alpha1/instascale.go delete mode 100644 client/applyconfiguration/codeflare/v1alpha1/instascalespec.go delete mode 100644 client/applyconfiguration/codeflare/v1alpha1/instascalestatus.go delete mode 100644 client/applyconfiguration/codeflare/v1alpha1/mcad.go delete mode 100644 client/applyconfiguration/codeflare/v1alpha1/mcadspec.go delete mode 100644 client/applyconfiguration/codeflare/v1alpha1/mcadstatus.go delete mode 100644 client/applyconfiguration/internal/internal.go delete mode 100644 client/applyconfiguration/utils.go delete mode 100644 client/clientset/versioned/clientset.go delete mode 100644 client/clientset/versioned/fake/clientset_generated.go delete mode 100644 client/clientset/versioned/fake/doc.go delete mode 100644 client/clientset/versioned/fake/register.go delete mode 100644 client/clientset/versioned/scheme/doc.go delete mode 100644 client/clientset/versioned/scheme/register.go delete mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go delete mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/doc.go delete mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/fake/doc.go delete mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go delete mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go delete mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go delete mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/generated_expansion.go delete mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go delete mode 100644 client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go delete mode 100644 client/informer/externalversions/codeflare/interface.go delete mode 100644 client/informer/externalversions/codeflare/v1alpha1/instascale.go delete mode 100644 client/informer/externalversions/codeflare/v1alpha1/interface.go delete mode 100644 client/informer/externalversions/codeflare/v1alpha1/mcad.go delete mode 100644 client/informer/externalversions/factory.go delete mode 100644 client/informer/externalversions/generic.go delete mode 100644 client/informer/externalversions/internalinterfaces/factory_interfaces.go delete mode 100644 client/listers/codeflare/v1alpha1/expansion_generated.go delete mode 100644 client/listers/codeflare/v1alpha1/instascale.go delete mode 100644 client/listers/codeflare/v1alpha1/mcad.go diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml index a5cc6ad2..304d9df0 100644 --- a/.github/workflows/verify_generated_files.yml +++ b/.github/workflows/verify_generated_files.yml @@ -26,17 +26,6 @@ jobs: - name: Verify that the DeepCopy, DeepCopyInto, and DeepCopyObject method implementations have been generated run: make generate && git diff --exit-code - verify-generate-client: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set Go - uses: actions/setup-go@v3 - with: - go-version: v1.19 - - name: Verify that the latest client has been generated - run: make generate-client && git diff --exit-code - verify-imports: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index eaf71aab..5e14740b 100644 --- a/Makefile +++ b/Makefile @@ -158,42 +158,6 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./api/..." -.PHONY: generate-client ## Generate client packages and organize the goimports -generate-client: generate-client-files imports - -.PHONY: generate-client-files -generate-client-files: code-generator - rm -rf client - $(APPLYCONFIGURATION_GEN) \ - --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ - --go-header-file="hack/boilerplate.go.txt" \ - --output-package="github.com/project-codeflare/codeflare-operator/client/applyconfiguration" \ - --output-base="." \ - --trim-path-prefix "github.com/project-codeflare/codeflare-operator" - $(CLIENT_GEN) \ - --input="codeflare/v1alpha1" \ - --input-base="github.com/project-codeflare/codeflare-operator/api" \ - --apply-configuration-package="github.com/project-codeflare/codeflare-operator/client/applyconfiguration" \ - --go-header-file="hack/boilerplate.go.txt" \ - --clientset-name "versioned" \ - --output-package="github.com/project-codeflare/codeflare-operator/client/clientset" \ - --output-base="." \ - --trim-path-prefix "github.com/project-codeflare/codeflare-operator" - $(LISTER_GEN) \ - --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ - --go-header-file="hack/boilerplate.go.txt" \ - --output-base="." \ - --output-package="github.com/project-codeflare/codeflare-operator/client/listers" \ - --trim-path-prefix "github.com/project-codeflare/codeflare-operator" - $(INFORMER_GEN) \ - --input-dirs="github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" \ - --versioned-clientset-package="github.com/project-codeflare/codeflare-operator/client/clientset/versioned" \ - --listers-package="github.com/project-codeflare/codeflare-operator/client/listers" \ - --go-header-file="hack/boilerplate.go.txt" \ - --output-base="." \ - --output-package="github.com/project-codeflare/codeflare-operator/client/informer" \ - --trim-path-prefix "github.com/project-codeflare/codeflare-operator" - .PHONY: fmt fmt: ## Run go fmt against code. go fmt ./... @@ -266,10 +230,6 @@ $(LOCALBIN): ## Tool Binaries KUSTOMIZE ?= $(LOCALBIN)/kustomize -APPLYCONFIGURATION_GEN ?= $(LOCALBIN)/applyconfiguration-gen -CLIENT_GEN ?= $(LOCALBIN)/client-gen -LISTER_GEN ?= $(LOCALBIN)/lister-gen -INFORMER_GEN ?= $(LOCALBIN)/informer-gen CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest OPENSHIFT-GOIMPORTS ?= $(LOCALBIN)/openshift-goimports @@ -300,29 +260,6 @@ $(GH_CLI): $(LOCALBIN) rm -rf $(GH_CLI_DL_FILENAME) rm $(GH_CLI_DL_FILENAME).tar.gz -.PHONY: code-generator -code-generator: $(APPLYCONFIGURATION_GEN) $(CLIENT_GEN) $(LISTER_GEN) $(INFORMER_GEN) - -.PHONY: applyconfiguration-gen -applyconfiguration-gen: $(APPLYCONFIGURATION_GEN) -$(APPLYCONFIGURATION_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/applyconfiguration-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/applyconfiguration-gen@$(CODEGEN_VERSION) - -.PHONY: client-gen -client-gen: $(CLIENT_GEN) -$(CLIENT_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/client-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/client-gen@$(CODEGEN_VERSION) - -.PHONY: lister-gen -lister-gen: $(LISTER_GEN) -$(LISTER_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/lister-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/lister-gen@$(CODEGEN_VERSION) - -.PHONY: informer-gen -informer-gen: $(INFORMER_GEN) -$(INFORMER_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/informer-gen || GOBIN=$(LOCALBIN) go install k8s.io/code-generator/cmd/informer-gen@$(CODEGEN_VERSION) - .PHONY: controller-gen controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. $(CONTROLLER_GEN): $(LOCALBIN) diff --git a/client/applyconfiguration/codeflare/v1alpha1/instascale.go b/client/applyconfiguration/codeflare/v1alpha1/instascale.go deleted file mode 100644 index 3cb4c8c6..00000000 --- a/client/applyconfiguration/codeflare/v1alpha1/instascale.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - v1 "k8s.io/client-go/applyconfigurations/meta/v1" -) - -// InstaScaleApplyConfiguration represents an declarative configuration of the InstaScale type for use -// with apply. -type InstaScaleApplyConfiguration struct { - v1.TypeMetaApplyConfiguration `json:",inline"` - *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` - Spec *InstaScaleSpecApplyConfiguration `json:"spec,omitempty"` - Status *InstaScaleStatusApplyConfiguration `json:"status,omitempty"` -} - -// InstaScale constructs an declarative configuration of the InstaScale type for use with -// apply. -func InstaScale(name, namespace string) *InstaScaleApplyConfiguration { - b := &InstaScaleApplyConfiguration{} - b.WithName(name) - b.WithNamespace(namespace) - b.WithKind("InstaScale") - b.WithAPIVersion("codeflare.codeflare.dev/v1alpha1") - return b -} - -// WithKind sets the Kind field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Kind field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithKind(value string) *InstaScaleApplyConfiguration { - b.Kind = &value - return b -} - -// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the APIVersion field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithAPIVersion(value string) *InstaScaleApplyConfiguration { - b.APIVersion = &value - return b -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithName(value string) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Name = &value - return b -} - -// WithGenerateName sets the GenerateName field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the GenerateName field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithGenerateName(value string) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.GenerateName = &value - return b -} - -// WithNamespace sets the Namespace field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Namespace field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithNamespace(value string) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Namespace = &value - return b -} - -// WithUID sets the UID field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UID field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithUID(value types.UID) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.UID = &value - return b -} - -// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ResourceVersion field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithResourceVersion(value string) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.ResourceVersion = &value - return b -} - -// WithGeneration sets the Generation field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Generation field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithGeneration(value int64) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Generation = &value - return b -} - -// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the CreationTimestamp field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithCreationTimestamp(value metav1.Time) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.CreationTimestamp = &value - return b -} - -// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionTimestamp field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionTimestamp = &value - return b -} - -// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionGracePeriodSeconds = &value - return b -} - -// WithLabels puts the entries into the Labels field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Labels field, -// overwriting an existing map entries in Labels field with the same key. -func (b *InstaScaleApplyConfiguration) WithLabels(entries map[string]string) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Labels == nil && len(entries) > 0 { - b.Labels = make(map[string]string, len(entries)) - } - for k, v := range entries { - b.Labels[k] = v - } - return b -} - -// WithAnnotations puts the entries into the Annotations field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Annotations field, -// overwriting an existing map entries in Annotations field with the same key. -func (b *InstaScaleApplyConfiguration) WithAnnotations(entries map[string]string) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Annotations == nil && len(entries) > 0 { - b.Annotations = make(map[string]string, len(entries)) - } - for k, v := range entries { - b.Annotations[k] = v - } - return b -} - -// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the OwnerReferences field. -func (b *InstaScaleApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - if values[i] == nil { - panic("nil value passed to WithOwnerReferences") - } - b.OwnerReferences = append(b.OwnerReferences, *values[i]) - } - return b -} - -// WithFinalizers adds the given value to the Finalizers field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Finalizers field. -func (b *InstaScaleApplyConfiguration) WithFinalizers(values ...string) *InstaScaleApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - b.Finalizers = append(b.Finalizers, values[i]) - } - return b -} - -func (b *InstaScaleApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { - if b.ObjectMetaApplyConfiguration == nil { - b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} - } -} - -// WithSpec sets the Spec field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Spec field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithSpec(value *InstaScaleSpecApplyConfiguration) *InstaScaleApplyConfiguration { - b.Spec = value - return b -} - -// WithStatus sets the Status field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Status field is set to the value of the last call. -func (b *InstaScaleApplyConfiguration) WithStatus(value *InstaScaleStatusApplyConfiguration) *InstaScaleApplyConfiguration { - b.Status = value - return b -} diff --git a/client/applyconfiguration/codeflare/v1alpha1/instascalespec.go b/client/applyconfiguration/codeflare/v1alpha1/instascalespec.go deleted file mode 100644 index e6ed5abd..00000000 --- a/client/applyconfiguration/codeflare/v1alpha1/instascalespec.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - v1 "k8s.io/api/core/v1" -) - -// InstaScaleSpecApplyConfiguration represents an declarative configuration of the InstaScaleSpec type for use -// with apply. -type InstaScaleSpecApplyConfiguration struct { - EnableMonitoring *bool `json:"enableMonitoring,omitempty"` - MaxScaleoutAllowed *int `json:"maxScaleoutAllowed,omitempty"` - UseMachinePools *bool `json:"useMachinePools,omitempty"` - ControllerResources *v1.ResourceRequirements `json:"controllerResources,omitempty"` - ControllerImage *string `json:"controllerImage,omitempty"` -} - -// InstaScaleSpecApplyConfiguration constructs an declarative configuration of the InstaScaleSpec type for use with -// apply. -func InstaScaleSpec() *InstaScaleSpecApplyConfiguration { - return &InstaScaleSpecApplyConfiguration{} -} - -// WithEnableMonitoring sets the EnableMonitoring field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the EnableMonitoring field is set to the value of the last call. -func (b *InstaScaleSpecApplyConfiguration) WithEnableMonitoring(value bool) *InstaScaleSpecApplyConfiguration { - b.EnableMonitoring = &value - return b -} - -// WithMaxScaleoutAllowed sets the MaxScaleoutAllowed field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the MaxScaleoutAllowed field is set to the value of the last call. -func (b *InstaScaleSpecApplyConfiguration) WithMaxScaleoutAllowed(value int) *InstaScaleSpecApplyConfiguration { - b.MaxScaleoutAllowed = &value - return b -} - -// WithUseMachinePools sets the UseMachinePools field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UseMachinePools field is set to the value of the last call. -func (b *InstaScaleSpecApplyConfiguration) WithUseMachinePools(value bool) *InstaScaleSpecApplyConfiguration { - b.UseMachinePools = &value - return b -} - -// WithControllerResources sets the ControllerResources field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ControllerResources field is set to the value of the last call. -func (b *InstaScaleSpecApplyConfiguration) WithControllerResources(value v1.ResourceRequirements) *InstaScaleSpecApplyConfiguration { - b.ControllerResources = &value - return b -} - -// WithControllerImage sets the ControllerImage field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ControllerImage field is set to the value of the last call. -func (b *InstaScaleSpecApplyConfiguration) WithControllerImage(value string) *InstaScaleSpecApplyConfiguration { - b.ControllerImage = &value - return b -} diff --git a/client/applyconfiguration/codeflare/v1alpha1/instascalestatus.go b/client/applyconfiguration/codeflare/v1alpha1/instascalestatus.go deleted file mode 100644 index 9643218f..00000000 --- a/client/applyconfiguration/codeflare/v1alpha1/instascalestatus.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -// InstaScaleStatusApplyConfiguration represents an declarative configuration of the InstaScaleStatus type for use -// with apply. -type InstaScaleStatusApplyConfiguration struct { - Ready *bool `json:"ready,omitempty"` -} - -// InstaScaleStatusApplyConfiguration constructs an declarative configuration of the InstaScaleStatus type for use with -// apply. -func InstaScaleStatus() *InstaScaleStatusApplyConfiguration { - return &InstaScaleStatusApplyConfiguration{} -} - -// WithReady sets the Ready field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Ready field is set to the value of the last call. -func (b *InstaScaleStatusApplyConfiguration) WithReady(value bool) *InstaScaleStatusApplyConfiguration { - b.Ready = &value - return b -} diff --git a/client/applyconfiguration/codeflare/v1alpha1/mcad.go b/client/applyconfiguration/codeflare/v1alpha1/mcad.go deleted file mode 100644 index 0b7a441b..00000000 --- a/client/applyconfiguration/codeflare/v1alpha1/mcad.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - v1 "k8s.io/client-go/applyconfigurations/meta/v1" -) - -// MCADApplyConfiguration represents an declarative configuration of the MCAD type for use -// with apply. -type MCADApplyConfiguration struct { - v1.TypeMetaApplyConfiguration `json:",inline"` - *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"` - Spec *MCADSpecApplyConfiguration `json:"spec,omitempty"` - Status *MCADStatusApplyConfiguration `json:"status,omitempty"` -} - -// MCAD constructs an declarative configuration of the MCAD type for use with -// apply. -func MCAD(name, namespace string) *MCADApplyConfiguration { - b := &MCADApplyConfiguration{} - b.WithName(name) - b.WithNamespace(namespace) - b.WithKind("MCAD") - b.WithAPIVersion("codeflare.codeflare.dev/v1alpha1") - return b -} - -// WithKind sets the Kind field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Kind field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithKind(value string) *MCADApplyConfiguration { - b.Kind = &value - return b -} - -// WithAPIVersion sets the APIVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the APIVersion field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithAPIVersion(value string) *MCADApplyConfiguration { - b.APIVersion = &value - return b -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithName(value string) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Name = &value - return b -} - -// WithGenerateName sets the GenerateName field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the GenerateName field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithGenerateName(value string) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.GenerateName = &value - return b -} - -// WithNamespace sets the Namespace field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Namespace field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithNamespace(value string) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Namespace = &value - return b -} - -// WithUID sets the UID field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the UID field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithUID(value types.UID) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.UID = &value - return b -} - -// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ResourceVersion field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithResourceVersion(value string) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.ResourceVersion = &value - return b -} - -// WithGeneration sets the Generation field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Generation field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithGeneration(value int64) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.Generation = &value - return b -} - -// WithCreationTimestamp sets the CreationTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the CreationTimestamp field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithCreationTimestamp(value metav1.Time) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.CreationTimestamp = &value - return b -} - -// WithDeletionTimestamp sets the DeletionTimestamp field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionTimestamp field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithDeletionTimestamp(value metav1.Time) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionTimestamp = &value - return b -} - -// WithDeletionGracePeriodSeconds sets the DeletionGracePeriodSeconds field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DeletionGracePeriodSeconds field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithDeletionGracePeriodSeconds(value int64) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - b.DeletionGracePeriodSeconds = &value - return b -} - -// WithLabels puts the entries into the Labels field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Labels field, -// overwriting an existing map entries in Labels field with the same key. -func (b *MCADApplyConfiguration) WithLabels(entries map[string]string) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Labels == nil && len(entries) > 0 { - b.Labels = make(map[string]string, len(entries)) - } - for k, v := range entries { - b.Labels[k] = v - } - return b -} - -// WithAnnotations puts the entries into the Annotations field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, the entries provided by each call will be put on the Annotations field, -// overwriting an existing map entries in Annotations field with the same key. -func (b *MCADApplyConfiguration) WithAnnotations(entries map[string]string) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - if b.Annotations == nil && len(entries) > 0 { - b.Annotations = make(map[string]string, len(entries)) - } - for k, v := range entries { - b.Annotations[k] = v - } - return b -} - -// WithOwnerReferences adds the given value to the OwnerReferences field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the OwnerReferences field. -func (b *MCADApplyConfiguration) WithOwnerReferences(values ...*v1.OwnerReferenceApplyConfiguration) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - if values[i] == nil { - panic("nil value passed to WithOwnerReferences") - } - b.OwnerReferences = append(b.OwnerReferences, *values[i]) - } - return b -} - -// WithFinalizers adds the given value to the Finalizers field in the declarative configuration -// and returns the receiver, so that objects can be build by chaining "With" function invocations. -// If called multiple times, values provided by each call will be appended to the Finalizers field. -func (b *MCADApplyConfiguration) WithFinalizers(values ...string) *MCADApplyConfiguration { - b.ensureObjectMetaApplyConfigurationExists() - for i := range values { - b.Finalizers = append(b.Finalizers, values[i]) - } - return b -} - -func (b *MCADApplyConfiguration) ensureObjectMetaApplyConfigurationExists() { - if b.ObjectMetaApplyConfiguration == nil { - b.ObjectMetaApplyConfiguration = &v1.ObjectMetaApplyConfiguration{} - } -} - -// WithSpec sets the Spec field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Spec field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithSpec(value *MCADSpecApplyConfiguration) *MCADApplyConfiguration { - b.Spec = value - return b -} - -// WithStatus sets the Status field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Status field is set to the value of the last call. -func (b *MCADApplyConfiguration) WithStatus(value *MCADStatusApplyConfiguration) *MCADApplyConfiguration { - b.Status = value - return b -} diff --git a/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go b/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go deleted file mode 100644 index 9358586b..00000000 --- a/client/applyconfiguration/codeflare/v1alpha1/mcadspec.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - v1 "k8s.io/api/core/v1" -) - -// MCADSpecApplyConfiguration represents an declarative configuration of the MCADSpec type for use -// with apply. -type MCADSpecApplyConfiguration struct { - EnableMonitoring *bool `json:"enableMonitoring,omitempty"` - MultiCluster *bool `json:"multiCluster,omitempty"` - DispatcherMode *bool `json:"dispatcherMode,omitempty"` - PreemptionEnabled *bool `json:"preemptionEnabled,omitempty"` - AgentConfigs *string `json:"agentConfigs,omitempty"` - QuotaRestURL *string `json:"quotaRestURL,omitempty"` - PodCreationTimeout *int `json:"podCreationTimeout,omitempty"` - ControllerResources *v1.ResourceRequirements `json:"controllerResources,omitempty"` - ControllerImage *string `json:"controllerImage,omitempty"` -} - -// MCADSpecApplyConfiguration constructs an declarative configuration of the MCADSpec type for use with -// apply. -func MCADSpec() *MCADSpecApplyConfiguration { - return &MCADSpecApplyConfiguration{} -} - -// WithEnableMonitoring sets the EnableMonitoring field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the EnableMonitoring field is set to the value of the last call. -func (b *MCADSpecApplyConfiguration) WithEnableMonitoring(value bool) *MCADSpecApplyConfiguration { - b.EnableMonitoring = &value - return b -} - -// WithMultiCluster sets the MultiCluster field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the MultiCluster field is set to the value of the last call. -func (b *MCADSpecApplyConfiguration) WithMultiCluster(value bool) *MCADSpecApplyConfiguration { - b.MultiCluster = &value - return b -} - -// WithDispatcherMode sets the DispatcherMode field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the DispatcherMode field is set to the value of the last call. -func (b *MCADSpecApplyConfiguration) WithDispatcherMode(value bool) *MCADSpecApplyConfiguration { - b.DispatcherMode = &value - return b -} - -// WithPreemptionEnabled sets the PreemptionEnabled field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the PreemptionEnabled field is set to the value of the last call. -func (b *MCADSpecApplyConfiguration) WithPreemptionEnabled(value bool) *MCADSpecApplyConfiguration { - b.PreemptionEnabled = &value - return b -} - -// WithAgentConfigs sets the AgentConfigs field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the AgentConfigs field is set to the value of the last call. -func (b *MCADSpecApplyConfiguration) WithAgentConfigs(value string) *MCADSpecApplyConfiguration { - b.AgentConfigs = &value - return b -} - -// WithQuotaRestURL sets the QuotaRestURL field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the QuotaRestURL field is set to the value of the last call. -func (b *MCADSpecApplyConfiguration) WithQuotaRestURL(value string) *MCADSpecApplyConfiguration { - b.QuotaRestURL = &value - return b -} - -// WithPodCreationTimeout sets the PodCreationTimeout field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the PodCreationTimeout field is set to the value of the last call. -func (b *MCADSpecApplyConfiguration) WithPodCreationTimeout(value int) *MCADSpecApplyConfiguration { - b.PodCreationTimeout = &value - return b -} - -// WithControllerResources sets the ControllerResources field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ControllerResources field is set to the value of the last call. -func (b *MCADSpecApplyConfiguration) WithControllerResources(value v1.ResourceRequirements) *MCADSpecApplyConfiguration { - b.ControllerResources = &value - return b -} - -// WithControllerImage sets the ControllerImage field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ControllerImage field is set to the value of the last call. -func (b *MCADSpecApplyConfiguration) WithControllerImage(value string) *MCADSpecApplyConfiguration { - b.ControllerImage = &value - return b -} diff --git a/client/applyconfiguration/codeflare/v1alpha1/mcadstatus.go b/client/applyconfiguration/codeflare/v1alpha1/mcadstatus.go deleted file mode 100644 index 0b93b070..00000000 --- a/client/applyconfiguration/codeflare/v1alpha1/mcadstatus.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -// MCADStatusApplyConfiguration represents an declarative configuration of the MCADStatus type for use -// with apply. -type MCADStatusApplyConfiguration struct { - Ready *bool `json:"ready,omitempty"` -} - -// MCADStatusApplyConfiguration constructs an declarative configuration of the MCADStatus type for use with -// apply. -func MCADStatus() *MCADStatusApplyConfiguration { - return &MCADStatusApplyConfiguration{} -} - -// WithReady sets the Ready field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Ready field is set to the value of the last call. -func (b *MCADStatusApplyConfiguration) WithReady(value bool) *MCADStatusApplyConfiguration { - b.Ready = &value - return b -} diff --git a/client/applyconfiguration/internal/internal.go b/client/applyconfiguration/internal/internal.go deleted file mode 100644 index e3bfb7d5..00000000 --- a/client/applyconfiguration/internal/internal.go +++ /dev/null @@ -1,62 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package internal - -import ( - "fmt" - "sync" - - typed "sigs.k8s.io/structured-merge-diff/v4/typed" -) - -func Parser() *typed.Parser { - parserOnce.Do(func() { - var err error - parser, err = typed.NewParser(schemaYAML) - if err != nil { - panic(fmt.Sprintf("Failed to parse schema: %v", err)) - } - }) - return parser -} - -var parserOnce sync.Once -var parser *typed.Parser -var schemaYAML = typed.YAMLObject(`types: -- name: __untyped_atomic_ - scalar: untyped - list: - elementType: - namedType: __untyped_atomic_ - elementRelationship: atomic - map: - elementType: - namedType: __untyped_atomic_ - elementRelationship: atomic -- name: __untyped_deduced_ - scalar: untyped - list: - elementType: - namedType: __untyped_atomic_ - elementRelationship: atomic - map: - elementType: - namedType: __untyped_deduced_ - elementRelationship: separable -`) diff --git a/client/applyconfiguration/utils.go b/client/applyconfiguration/utils.go deleted file mode 100644 index 44a5e7aa..00000000 --- a/client/applyconfiguration/utils.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package applyconfiguration - -import ( - schema "k8s.io/apimachinery/pkg/runtime/schema" - - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" -) - -// ForKind returns an apply configuration type for the given GroupVersionKind, or nil if no -// apply configuration type exists for the given GroupVersionKind. -func ForKind(kind schema.GroupVersionKind) interface{} { - switch kind { - // Group=codeflare.codeflare.dev, Version=v1alpha1 - case v1alpha1.SchemeGroupVersion.WithKind("InstaScale"): - return &codeflarev1alpha1.InstaScaleApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("InstaScaleSpec"): - return &codeflarev1alpha1.InstaScaleSpecApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("InstaScaleStatus"): - return &codeflarev1alpha1.InstaScaleStatusApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("MCAD"): - return &codeflarev1alpha1.MCADApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("MCADSpec"): - return &codeflarev1alpha1.MCADSpecApplyConfiguration{} - case v1alpha1.SchemeGroupVersion.WithKind("MCADStatus"): - return &codeflarev1alpha1.MCADStatusApplyConfiguration{} - - } - return nil -} diff --git a/client/clientset/versioned/clientset.go b/client/clientset/versioned/clientset.go deleted file mode 100644 index ca6d68fb..00000000 --- a/client/clientset/versioned/clientset.go +++ /dev/null @@ -1,121 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package versioned - -import ( - "fmt" - "net/http" - - discovery "k8s.io/client-go/discovery" - rest "k8s.io/client-go/rest" - flowcontrol "k8s.io/client-go/util/flowcontrol" - - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" -) - -type Interface interface { - Discovery() discovery.DiscoveryInterface - CodeflareV1alpha1() codeflarev1alpha1.CodeflareV1alpha1Interface -} - -// Clientset contains the clients for groups. -type Clientset struct { - *discovery.DiscoveryClient - codeflareV1alpha1 *codeflarev1alpha1.CodeflareV1alpha1Client -} - -// CodeflareV1alpha1 retrieves the CodeflareV1alpha1Client -func (c *Clientset) CodeflareV1alpha1() codeflarev1alpha1.CodeflareV1alpha1Interface { - return c.codeflareV1alpha1 -} - -// Discovery retrieves the DiscoveryClient -func (c *Clientset) Discovery() discovery.DiscoveryInterface { - if c == nil { - return nil - } - return c.DiscoveryClient -} - -// NewForConfig creates a new Clientset for the given config. -// If config's RateLimiter is not set and QPS and Burst are acceptable, -// NewForConfig will generate a rate-limiter in configShallowCopy. -// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), -// where httpClient was generated with rest.HTTPClientFor(c). -func NewForConfig(c *rest.Config) (*Clientset, error) { - configShallowCopy := *c - - if configShallowCopy.UserAgent == "" { - configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() - } - - // share the transport between all clients - httpClient, err := rest.HTTPClientFor(&configShallowCopy) - if err != nil { - return nil, err - } - - return NewForConfigAndClient(&configShallowCopy, httpClient) -} - -// NewForConfigAndClient creates a new Clientset for the given config and http client. -// Note the http client provided takes precedence over the configured transport values. -// If config's RateLimiter is not set and QPS and Burst are acceptable, -// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. -func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { - configShallowCopy := *c - if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { - if configShallowCopy.Burst <= 0 { - return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") - } - configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) - } - - var cs Clientset - var err error - cs.codeflareV1alpha1, err = codeflarev1alpha1.NewForConfigAndClient(&configShallowCopy, httpClient) - if err != nil { - return nil, err - } - - cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) - if err != nil { - return nil, err - } - return &cs, nil -} - -// NewForConfigOrDie creates a new Clientset for the given config and -// panics if there is an error in the config. -func NewForConfigOrDie(c *rest.Config) *Clientset { - cs, err := NewForConfig(c) - if err != nil { - panic(err) - } - return cs -} - -// New creates a new Clientset for the given RESTClient. -func New(c rest.Interface) *Clientset { - var cs Clientset - cs.codeflareV1alpha1 = codeflarev1alpha1.New(c) - - cs.DiscoveryClient = discovery.NewDiscoveryClient(c) - return &cs -} diff --git a/client/clientset/versioned/fake/clientset_generated.go b/client/clientset/versioned/fake/clientset_generated.go deleted file mode 100644 index b3e9cbb9..00000000 --- a/client/clientset/versioned/fake/clientset_generated.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/discovery" - fakediscovery "k8s.io/client-go/discovery/fake" - "k8s.io/client-go/testing" - - clientset "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" - fakecodeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1/fake" -) - -// NewSimpleClientset returns a clientset that will respond with the provided objects. -// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, -// without applying any validations and/or defaults. It shouldn't be considered a replacement -// for a real clientset and is mostly useful in simple unit tests. -func NewSimpleClientset(objects ...runtime.Object) *Clientset { - o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) - for _, obj := range objects { - if err := o.Add(obj); err != nil { - panic(err) - } - } - - cs := &Clientset{tracker: o} - cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} - cs.AddReactor("*", "*", testing.ObjectReaction(o)) - cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { - gvr := action.GetResource() - ns := action.GetNamespace() - watch, err := o.Watch(gvr, ns) - if err != nil { - return false, nil, err - } - return true, watch, nil - }) - - return cs -} - -// Clientset implements clientset.Interface. Meant to be embedded into a -// struct to get a default implementation. This makes faking out just the method -// you want to test easier. -type Clientset struct { - testing.Fake - discovery *fakediscovery.FakeDiscovery - tracker testing.ObjectTracker -} - -func (c *Clientset) Discovery() discovery.DiscoveryInterface { - return c.discovery -} - -func (c *Clientset) Tracker() testing.ObjectTracker { - return c.tracker -} - -var ( - _ clientset.Interface = &Clientset{} - _ testing.FakeClient = &Clientset{} -) - -// CodeflareV1alpha1 retrieves the CodeflareV1alpha1Client -func (c *Clientset) CodeflareV1alpha1() codeflarev1alpha1.CodeflareV1alpha1Interface { - return &fakecodeflarev1alpha1.FakeCodeflareV1alpha1{Fake: &c.Fake} -} diff --git a/client/clientset/versioned/fake/doc.go b/client/clientset/versioned/fake/doc.go deleted file mode 100644 index 86aee2a9..00000000 --- a/client/clientset/versioned/fake/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -// This package has the automatically generated fake clientset. -package fake diff --git a/client/clientset/versioned/fake/register.go b/client/clientset/versioned/fake/register.go deleted file mode 100644 index 05244eb5..00000000 --- a/client/clientset/versioned/fake/register.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - schema "k8s.io/apimachinery/pkg/runtime/schema" - serializer "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -var scheme = runtime.NewScheme() -var codecs = serializer.NewCodecFactory(scheme) - -var localSchemeBuilder = runtime.SchemeBuilder{ - codeflarev1alpha1.AddToScheme, -} - -// AddToScheme adds all types of this clientset into the given scheme. This allows composition -// of clientsets, like in: -// -// import ( -// "k8s.io/client-go/kubernetes" -// clientsetscheme "k8s.io/client-go/kubernetes/scheme" -// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" -// ) -// -// kclientset, _ := kubernetes.NewForConfig(c) -// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) -// -// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types -// correctly. -var AddToScheme = localSchemeBuilder.AddToScheme - -func init() { - v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) - utilruntime.Must(AddToScheme(scheme)) -} diff --git a/client/clientset/versioned/scheme/doc.go b/client/clientset/versioned/scheme/doc.go deleted file mode 100644 index 5f6297fd..00000000 --- a/client/clientset/versioned/scheme/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -// This package contains the scheme of the automatically generated clientset. -package scheme diff --git a/client/clientset/versioned/scheme/register.go b/client/clientset/versioned/scheme/register.go deleted file mode 100644 index 1a52d7da..00000000 --- a/client/clientset/versioned/scheme/register.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package scheme - -import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - schema "k8s.io/apimachinery/pkg/runtime/schema" - serializer "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -var Scheme = runtime.NewScheme() -var Codecs = serializer.NewCodecFactory(Scheme) -var ParameterCodec = runtime.NewParameterCodec(Scheme) -var localSchemeBuilder = runtime.SchemeBuilder{ - codeflarev1alpha1.AddToScheme, -} - -// AddToScheme adds all types of this clientset into the given scheme. This allows composition -// of clientsets, like in: -// -// import ( -// "k8s.io/client-go/kubernetes" -// clientsetscheme "k8s.io/client-go/kubernetes/scheme" -// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" -// ) -// -// kclientset, _ := kubernetes.NewForConfig(c) -// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) -// -// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types -// correctly. -var AddToScheme = localSchemeBuilder.AddToScheme - -func init() { - v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) - utilruntime.Must(AddToScheme(Scheme)) -} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go b/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go deleted file mode 100644 index bbe38940..00000000 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/codeflare_client.go +++ /dev/null @@ -1,113 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "net/http" - - rest "k8s.io/client-go/rest" - - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" -) - -type CodeflareV1alpha1Interface interface { - RESTClient() rest.Interface - InstaScalesGetter - MCADsGetter -} - -// CodeflareV1alpha1Client is used to interact with features provided by the codeflare.codeflare.dev group. -type CodeflareV1alpha1Client struct { - restClient rest.Interface -} - -func (c *CodeflareV1alpha1Client) InstaScales(namespace string) InstaScaleInterface { - return newInstaScales(c, namespace) -} - -func (c *CodeflareV1alpha1Client) MCADs(namespace string) MCADInterface { - return newMCADs(c, namespace) -} - -// NewForConfig creates a new CodeflareV1alpha1Client for the given config. -// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), -// where httpClient was generated with rest.HTTPClientFor(c). -func NewForConfig(c *rest.Config) (*CodeflareV1alpha1Client, error) { - config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } - httpClient, err := rest.HTTPClientFor(&config) - if err != nil { - return nil, err - } - return NewForConfigAndClient(&config, httpClient) -} - -// NewForConfigAndClient creates a new CodeflareV1alpha1Client for the given config and http client. -// Note the http client provided takes precedence over the configured transport values. -func NewForConfigAndClient(c *rest.Config, h *http.Client) (*CodeflareV1alpha1Client, error) { - config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } - client, err := rest.RESTClientForConfigAndClient(&config, h) - if err != nil { - return nil, err - } - return &CodeflareV1alpha1Client{client}, nil -} - -// NewForConfigOrDie creates a new CodeflareV1alpha1Client for the given config and -// panics if there is an error in the config. -func NewForConfigOrDie(c *rest.Config) *CodeflareV1alpha1Client { - client, err := NewForConfig(c) - if err != nil { - panic(err) - } - return client -} - -// New creates a new CodeflareV1alpha1Client for the given RESTClient. -func New(c rest.Interface) *CodeflareV1alpha1Client { - return &CodeflareV1alpha1Client{c} -} - -func setConfigDefaults(config *rest.Config) error { - gv := v1alpha1.SchemeGroupVersion - config.GroupVersion = &gv - config.APIPath = "/apis" - config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() - - if config.UserAgent == "" { - config.UserAgent = rest.DefaultKubernetesUserAgent() - } - - return nil -} - -// RESTClient returns a RESTClient that is used to communicate -// with API server by this client implementation. -func (c *CodeflareV1alpha1Client) RESTClient() rest.Interface { - if c == nil { - return nil - } - return c.restClient -} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/doc.go b/client/clientset/versioned/typed/codeflare/v1alpha1/doc.go deleted file mode 100644 index cf7c1520..00000000 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -// This package has the automatically generated typed clients. -package v1alpha1 diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/doc.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/doc.go deleted file mode 100644 index 7c26ebe2..00000000 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -// Package fake has the automatically generated clients. -package fake diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go deleted file mode 100644 index 0a18ef5d..00000000 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_codeflare_client.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - rest "k8s.io/client-go/rest" - testing "k8s.io/client-go/testing" - - v1alpha1 "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/typed/codeflare/v1alpha1" -) - -type FakeCodeflareV1alpha1 struct { - *testing.Fake -} - -func (c *FakeCodeflareV1alpha1) InstaScales(namespace string) v1alpha1.InstaScaleInterface { - return &FakeInstaScales{c, namespace} -} - -func (c *FakeCodeflareV1alpha1) MCADs(namespace string) v1alpha1.MCADInterface { - return &FakeMCADs{c, namespace} -} - -// RESTClient returns a RESTClient that is used to communicate -// with API server by this client implementation. -func (c *FakeCodeflareV1alpha1) RESTClient() rest.Interface { - var ret *rest.RESTClient - return ret -} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go deleted file mode 100644 index fd6bced0..00000000 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_instascale.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - "context" - json "encoding/json" - "fmt" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" - - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" -) - -// FakeInstaScales implements InstaScaleInterface -type FakeInstaScales struct { - Fake *FakeCodeflareV1alpha1 - ns string -} - -var instascalesResource = v1alpha1.SchemeGroupVersion.WithResource("instascales") - -var instascalesKind = v1alpha1.SchemeGroupVersion.WithKind("InstaScale") - -// Get takes name of the instaScale, and returns the corresponding instaScale object, and an error if there is any. -func (c *FakeInstaScales) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.InstaScale, err error) { - obj, err := c.Fake. - Invokes(testing.NewGetAction(instascalesResource, c.ns, name), &v1alpha1.InstaScale{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.InstaScale), err -} - -// List takes label and field selectors, and returns the list of InstaScales that match those selectors. -func (c *FakeInstaScales) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.InstaScaleList, err error) { - obj, err := c.Fake. - Invokes(testing.NewListAction(instascalesResource, instascalesKind, c.ns, opts), &v1alpha1.InstaScaleList{}) - - if obj == nil { - return nil, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &v1alpha1.InstaScaleList{ListMeta: obj.(*v1alpha1.InstaScaleList).ListMeta} - for _, item := range obj.(*v1alpha1.InstaScaleList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested instaScales. -func (c *FakeInstaScales) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewWatchAction(instascalesResource, c.ns, opts)) - -} - -// Create takes the representation of a instaScale and creates it. Returns the server's representation of the instaScale, and an error, if there is any. -func (c *FakeInstaScales) Create(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.CreateOptions) (result *v1alpha1.InstaScale, err error) { - obj, err := c.Fake. - Invokes(testing.NewCreateAction(instascalesResource, c.ns, instaScale), &v1alpha1.InstaScale{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.InstaScale), err -} - -// Update takes the representation of a instaScale and updates it. Returns the server's representation of the instaScale, and an error, if there is any. -func (c *FakeInstaScales) Update(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.UpdateOptions) (result *v1alpha1.InstaScale, err error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateAction(instascalesResource, c.ns, instaScale), &v1alpha1.InstaScale{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.InstaScale), err -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakeInstaScales) UpdateStatus(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.UpdateOptions) (*v1alpha1.InstaScale, error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateSubresourceAction(instascalesResource, "status", c.ns, instaScale), &v1alpha1.InstaScale{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.InstaScale), err -} - -// Delete takes name of the instaScale and deletes it. Returns an error if one occurs. -func (c *FakeInstaScales) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewDeleteActionWithOptions(instascalesResource, c.ns, name, opts), &v1alpha1.InstaScale{}) - - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakeInstaScales) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewDeleteCollectionAction(instascalesResource, c.ns, listOpts) - - _, err := c.Fake.Invokes(action, &v1alpha1.InstaScaleList{}) - return err -} - -// Patch applies the patch and returns the patched instaScale. -func (c *FakeInstaScales) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.InstaScale, err error) { - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(instascalesResource, c.ns, name, pt, data, subresources...), &v1alpha1.InstaScale{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.InstaScale), err -} - -// Apply takes the given apply declarative configuration, applies it and returns the applied instaScale. -func (c *FakeInstaScales) Apply(ctx context.Context, instaScale *codeflarev1alpha1.InstaScaleApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.InstaScale, err error) { - if instaScale == nil { - return nil, fmt.Errorf("instaScale provided to Apply must not be nil") - } - data, err := json.Marshal(instaScale) - if err != nil { - return nil, err - } - name := instaScale.Name - if name == nil { - return nil, fmt.Errorf("instaScale.Name must be provided to Apply") - } - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(instascalesResource, c.ns, *name, types.ApplyPatchType, data), &v1alpha1.InstaScale{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.InstaScale), err -} - -// ApplyStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating ApplyStatus(). -func (c *FakeInstaScales) ApplyStatus(ctx context.Context, instaScale *codeflarev1alpha1.InstaScaleApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.InstaScale, err error) { - if instaScale == nil { - return nil, fmt.Errorf("instaScale provided to Apply must not be nil") - } - data, err := json.Marshal(instaScale) - if err != nil { - return nil, err - } - name := instaScale.Name - if name == nil { - return nil, fmt.Errorf("instaScale.Name must be provided to Apply") - } - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(instascalesResource, c.ns, *name, types.ApplyPatchType, data, "status"), &v1alpha1.InstaScale{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.InstaScale), err -} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go b/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go deleted file mode 100644 index a06c229a..00000000 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/fake/fake_mcad.go +++ /dev/null @@ -1,190 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - "context" - json "encoding/json" - "fmt" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" - - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" -) - -// FakeMCADs implements MCADInterface -type FakeMCADs struct { - Fake *FakeCodeflareV1alpha1 - ns string -} - -var mcadsResource = v1alpha1.SchemeGroupVersion.WithResource("mcads") - -var mcadsKind = v1alpha1.SchemeGroupVersion.WithKind("MCAD") - -// Get takes name of the mCAD, and returns the corresponding mCAD object, and an error if there is any. -func (c *FakeMCADs) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.MCAD, err error) { - obj, err := c.Fake. - Invokes(testing.NewGetAction(mcadsResource, c.ns, name), &v1alpha1.MCAD{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.MCAD), err -} - -// List takes label and field selectors, and returns the list of MCADs that match those selectors. -func (c *FakeMCADs) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.MCADList, err error) { - obj, err := c.Fake. - Invokes(testing.NewListAction(mcadsResource, mcadsKind, c.ns, opts), &v1alpha1.MCADList{}) - - if obj == nil { - return nil, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &v1alpha1.MCADList{ListMeta: obj.(*v1alpha1.MCADList).ListMeta} - for _, item := range obj.(*v1alpha1.MCADList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested mCADs. -func (c *FakeMCADs) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewWatchAction(mcadsResource, c.ns, opts)) - -} - -// Create takes the representation of a mCAD and creates it. Returns the server's representation of the mCAD, and an error, if there is any. -func (c *FakeMCADs) Create(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.CreateOptions) (result *v1alpha1.MCAD, err error) { - obj, err := c.Fake. - Invokes(testing.NewCreateAction(mcadsResource, c.ns, mCAD), &v1alpha1.MCAD{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.MCAD), err -} - -// Update takes the representation of a mCAD and updates it. Returns the server's representation of the mCAD, and an error, if there is any. -func (c *FakeMCADs) Update(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.UpdateOptions) (result *v1alpha1.MCAD, err error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateAction(mcadsResource, c.ns, mCAD), &v1alpha1.MCAD{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.MCAD), err -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakeMCADs) UpdateStatus(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.UpdateOptions) (*v1alpha1.MCAD, error) { - obj, err := c.Fake. - Invokes(testing.NewUpdateSubresourceAction(mcadsResource, "status", c.ns, mCAD), &v1alpha1.MCAD{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.MCAD), err -} - -// Delete takes name of the mCAD and deletes it. Returns an error if one occurs. -func (c *FakeMCADs) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewDeleteActionWithOptions(mcadsResource, c.ns, name, opts), &v1alpha1.MCAD{}) - - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakeMCADs) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewDeleteCollectionAction(mcadsResource, c.ns, listOpts) - - _, err := c.Fake.Invokes(action, &v1alpha1.MCADList{}) - return err -} - -// Patch applies the patch and returns the patched mCAD. -func (c *FakeMCADs) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.MCAD, err error) { - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(mcadsResource, c.ns, name, pt, data, subresources...), &v1alpha1.MCAD{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.MCAD), err -} - -// Apply takes the given apply declarative configuration, applies it and returns the applied mCAD. -func (c *FakeMCADs) Apply(ctx context.Context, mCAD *codeflarev1alpha1.MCADApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.MCAD, err error) { - if mCAD == nil { - return nil, fmt.Errorf("mCAD provided to Apply must not be nil") - } - data, err := json.Marshal(mCAD) - if err != nil { - return nil, err - } - name := mCAD.Name - if name == nil { - return nil, fmt.Errorf("mCAD.Name must be provided to Apply") - } - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(mcadsResource, c.ns, *name, types.ApplyPatchType, data), &v1alpha1.MCAD{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.MCAD), err -} - -// ApplyStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating ApplyStatus(). -func (c *FakeMCADs) ApplyStatus(ctx context.Context, mCAD *codeflarev1alpha1.MCADApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.MCAD, err error) { - if mCAD == nil { - return nil, fmt.Errorf("mCAD provided to Apply must not be nil") - } - data, err := json.Marshal(mCAD) - if err != nil { - return nil, err - } - name := mCAD.Name - if name == nil { - return nil, fmt.Errorf("mCAD.Name must be provided to Apply") - } - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceAction(mcadsResource, c.ns, *name, types.ApplyPatchType, data, "status"), &v1alpha1.MCAD{}) - - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.MCAD), err -} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/generated_expansion.go b/client/clientset/versioned/typed/codeflare/v1alpha1/generated_expansion.go deleted file mode 100644 index 4727cc4d..00000000 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/generated_expansion.go +++ /dev/null @@ -1,23 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package v1alpha1 - -type InstaScaleExpansion interface{} - -type MCADExpansion interface{} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go b/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go deleted file mode 100644 index 73cc95d3..00000000 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/instascale.go +++ /dev/null @@ -1,257 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "context" - json "encoding/json" - "fmt" - "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - rest "k8s.io/client-go/rest" - - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" - scheme "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" -) - -// InstaScalesGetter has a method to return a InstaScaleInterface. -// A group's client should implement this interface. -type InstaScalesGetter interface { - InstaScales(namespace string) InstaScaleInterface -} - -// InstaScaleInterface has methods to work with InstaScale resources. -type InstaScaleInterface interface { - Create(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.CreateOptions) (*v1alpha1.InstaScale, error) - Update(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.UpdateOptions) (*v1alpha1.InstaScale, error) - UpdateStatus(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.UpdateOptions) (*v1alpha1.InstaScale, error) - Delete(ctx context.Context, name string, opts v1.DeleteOptions) error - DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error - Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.InstaScale, error) - List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.InstaScaleList, error) - Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.InstaScale, err error) - Apply(ctx context.Context, instaScale *codeflarev1alpha1.InstaScaleApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.InstaScale, err error) - ApplyStatus(ctx context.Context, instaScale *codeflarev1alpha1.InstaScaleApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.InstaScale, err error) - InstaScaleExpansion -} - -// instaScales implements InstaScaleInterface -type instaScales struct { - client rest.Interface - ns string -} - -// newInstaScales returns a InstaScales -func newInstaScales(c *CodeflareV1alpha1Client, namespace string) *instaScales { - return &instaScales{ - client: c.RESTClient(), - ns: namespace, - } -} - -// Get takes name of the instaScale, and returns the corresponding instaScale object, and an error if there is any. -func (c *instaScales) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.InstaScale, err error) { - result = &v1alpha1.InstaScale{} - err = c.client.Get(). - Namespace(c.ns). - Resource("instascales"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} - -// List takes label and field selectors, and returns the list of InstaScales that match those selectors. -func (c *instaScales) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.InstaScaleList, err error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - result = &v1alpha1.InstaScaleList{} - err = c.client.Get(). - Namespace(c.ns). - Resource("instascales"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Do(ctx). - Into(result) - return -} - -// Watch returns a watch.Interface that watches the requested instaScales. -func (c *instaScales) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - opts.Watch = true - return c.client.Get(). - Namespace(c.ns). - Resource("instascales"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Watch(ctx) -} - -// Create takes the representation of a instaScale and creates it. Returns the server's representation of the instaScale, and an error, if there is any. -func (c *instaScales) Create(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.CreateOptions) (result *v1alpha1.InstaScale, err error) { - result = &v1alpha1.InstaScale{} - err = c.client.Post(). - Namespace(c.ns). - Resource("instascales"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(instaScale). - Do(ctx). - Into(result) - return -} - -// Update takes the representation of a instaScale and updates it. Returns the server's representation of the instaScale, and an error, if there is any. -func (c *instaScales) Update(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.UpdateOptions) (result *v1alpha1.InstaScale, err error) { - result = &v1alpha1.InstaScale{} - err = c.client.Put(). - Namespace(c.ns). - Resource("instascales"). - Name(instaScale.Name). - VersionedParams(&opts, scheme.ParameterCodec). - Body(instaScale). - Do(ctx). - Into(result) - return -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *instaScales) UpdateStatus(ctx context.Context, instaScale *v1alpha1.InstaScale, opts v1.UpdateOptions) (result *v1alpha1.InstaScale, err error) { - result = &v1alpha1.InstaScale{} - err = c.client.Put(). - Namespace(c.ns). - Resource("instascales"). - Name(instaScale.Name). - SubResource("status"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(instaScale). - Do(ctx). - Into(result) - return -} - -// Delete takes name of the instaScale and deletes it. Returns an error if one occurs. -func (c *instaScales) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - return c.client.Delete(). - Namespace(c.ns). - Resource("instascales"). - Name(name). - Body(&opts). - Do(ctx). - Error() -} - -// DeleteCollection deletes a collection of objects. -func (c *instaScales) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - var timeout time.Duration - if listOpts.TimeoutSeconds != nil { - timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second - } - return c.client.Delete(). - Namespace(c.ns). - Resource("instascales"). - VersionedParams(&listOpts, scheme.ParameterCodec). - Timeout(timeout). - Body(&opts). - Do(ctx). - Error() -} - -// Patch applies the patch and returns the patched instaScale. -func (c *instaScales) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.InstaScale, err error) { - result = &v1alpha1.InstaScale{} - err = c.client.Patch(pt). - Namespace(c.ns). - Resource("instascales"). - Name(name). - SubResource(subresources...). - VersionedParams(&opts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} - -// Apply takes the given apply declarative configuration, applies it and returns the applied instaScale. -func (c *instaScales) Apply(ctx context.Context, instaScale *codeflarev1alpha1.InstaScaleApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.InstaScale, err error) { - if instaScale == nil { - return nil, fmt.Errorf("instaScale provided to Apply must not be nil") - } - patchOpts := opts.ToPatchOptions() - data, err := json.Marshal(instaScale) - if err != nil { - return nil, err - } - name := instaScale.Name - if name == nil { - return nil, fmt.Errorf("instaScale.Name must be provided to Apply") - } - result = &v1alpha1.InstaScale{} - err = c.client.Patch(types.ApplyPatchType). - Namespace(c.ns). - Resource("instascales"). - Name(*name). - VersionedParams(&patchOpts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} - -// ApplyStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating ApplyStatus(). -func (c *instaScales) ApplyStatus(ctx context.Context, instaScale *codeflarev1alpha1.InstaScaleApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.InstaScale, err error) { - if instaScale == nil { - return nil, fmt.Errorf("instaScale provided to Apply must not be nil") - } - patchOpts := opts.ToPatchOptions() - data, err := json.Marshal(instaScale) - if err != nil { - return nil, err - } - - name := instaScale.Name - if name == nil { - return nil, fmt.Errorf("instaScale.Name must be provided to Apply") - } - - result = &v1alpha1.InstaScale{} - err = c.client.Patch(types.ApplyPatchType). - Namespace(c.ns). - Resource("instascales"). - Name(*name). - SubResource("status"). - VersionedParams(&patchOpts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} diff --git a/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go b/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go deleted file mode 100644 index 23587f12..00000000 --- a/client/clientset/versioned/typed/codeflare/v1alpha1/mcad.go +++ /dev/null @@ -1,257 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "context" - json "encoding/json" - "fmt" - "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - rest "k8s.io/client-go/rest" - - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/client/applyconfiguration/codeflare/v1alpha1" - scheme "github.com/project-codeflare/codeflare-operator/client/clientset/versioned/scheme" -) - -// MCADsGetter has a method to return a MCADInterface. -// A group's client should implement this interface. -type MCADsGetter interface { - MCADs(namespace string) MCADInterface -} - -// MCADInterface has methods to work with MCAD resources. -type MCADInterface interface { - Create(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.CreateOptions) (*v1alpha1.MCAD, error) - Update(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.UpdateOptions) (*v1alpha1.MCAD, error) - UpdateStatus(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.UpdateOptions) (*v1alpha1.MCAD, error) - Delete(ctx context.Context, name string, opts v1.DeleteOptions) error - DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error - Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.MCAD, error) - List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.MCADList, error) - Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.MCAD, err error) - Apply(ctx context.Context, mCAD *codeflarev1alpha1.MCADApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.MCAD, err error) - ApplyStatus(ctx context.Context, mCAD *codeflarev1alpha1.MCADApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.MCAD, err error) - MCADExpansion -} - -// mCADs implements MCADInterface -type mCADs struct { - client rest.Interface - ns string -} - -// newMCADs returns a MCADs -func newMCADs(c *CodeflareV1alpha1Client, namespace string) *mCADs { - return &mCADs{ - client: c.RESTClient(), - ns: namespace, - } -} - -// Get takes name of the mCAD, and returns the corresponding mCAD object, and an error if there is any. -func (c *mCADs) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.MCAD, err error) { - result = &v1alpha1.MCAD{} - err = c.client.Get(). - Namespace(c.ns). - Resource("mcads"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} - -// List takes label and field selectors, and returns the list of MCADs that match those selectors. -func (c *mCADs) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.MCADList, err error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - result = &v1alpha1.MCADList{} - err = c.client.Get(). - Namespace(c.ns). - Resource("mcads"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Do(ctx). - Into(result) - return -} - -// Watch returns a watch.Interface that watches the requested mCADs. -func (c *mCADs) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - opts.Watch = true - return c.client.Get(). - Namespace(c.ns). - Resource("mcads"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Watch(ctx) -} - -// Create takes the representation of a mCAD and creates it. Returns the server's representation of the mCAD, and an error, if there is any. -func (c *mCADs) Create(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.CreateOptions) (result *v1alpha1.MCAD, err error) { - result = &v1alpha1.MCAD{} - err = c.client.Post(). - Namespace(c.ns). - Resource("mcads"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(mCAD). - Do(ctx). - Into(result) - return -} - -// Update takes the representation of a mCAD and updates it. Returns the server's representation of the mCAD, and an error, if there is any. -func (c *mCADs) Update(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.UpdateOptions) (result *v1alpha1.MCAD, err error) { - result = &v1alpha1.MCAD{} - err = c.client.Put(). - Namespace(c.ns). - Resource("mcads"). - Name(mCAD.Name). - VersionedParams(&opts, scheme.ParameterCodec). - Body(mCAD). - Do(ctx). - Into(result) - return -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *mCADs) UpdateStatus(ctx context.Context, mCAD *v1alpha1.MCAD, opts v1.UpdateOptions) (result *v1alpha1.MCAD, err error) { - result = &v1alpha1.MCAD{} - err = c.client.Put(). - Namespace(c.ns). - Resource("mcads"). - Name(mCAD.Name). - SubResource("status"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(mCAD). - Do(ctx). - Into(result) - return -} - -// Delete takes name of the mCAD and deletes it. Returns an error if one occurs. -func (c *mCADs) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - return c.client.Delete(). - Namespace(c.ns). - Resource("mcads"). - Name(name). - Body(&opts). - Do(ctx). - Error() -} - -// DeleteCollection deletes a collection of objects. -func (c *mCADs) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - var timeout time.Duration - if listOpts.TimeoutSeconds != nil { - timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second - } - return c.client.Delete(). - Namespace(c.ns). - Resource("mcads"). - VersionedParams(&listOpts, scheme.ParameterCodec). - Timeout(timeout). - Body(&opts). - Do(ctx). - Error() -} - -// Patch applies the patch and returns the patched mCAD. -func (c *mCADs) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.MCAD, err error) { - result = &v1alpha1.MCAD{} - err = c.client.Patch(pt). - Namespace(c.ns). - Resource("mcads"). - Name(name). - SubResource(subresources...). - VersionedParams(&opts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} - -// Apply takes the given apply declarative configuration, applies it and returns the applied mCAD. -func (c *mCADs) Apply(ctx context.Context, mCAD *codeflarev1alpha1.MCADApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.MCAD, err error) { - if mCAD == nil { - return nil, fmt.Errorf("mCAD provided to Apply must not be nil") - } - patchOpts := opts.ToPatchOptions() - data, err := json.Marshal(mCAD) - if err != nil { - return nil, err - } - name := mCAD.Name - if name == nil { - return nil, fmt.Errorf("mCAD.Name must be provided to Apply") - } - result = &v1alpha1.MCAD{} - err = c.client.Patch(types.ApplyPatchType). - Namespace(c.ns). - Resource("mcads"). - Name(*name). - VersionedParams(&patchOpts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} - -// ApplyStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating ApplyStatus(). -func (c *mCADs) ApplyStatus(ctx context.Context, mCAD *codeflarev1alpha1.MCADApplyConfiguration, opts v1.ApplyOptions) (result *v1alpha1.MCAD, err error) { - if mCAD == nil { - return nil, fmt.Errorf("mCAD provided to Apply must not be nil") - } - patchOpts := opts.ToPatchOptions() - data, err := json.Marshal(mCAD) - if err != nil { - return nil, err - } - - name := mCAD.Name - if name == nil { - return nil, fmt.Errorf("mCAD.Name must be provided to Apply") - } - - result = &v1alpha1.MCAD{} - err = c.client.Patch(types.ApplyPatchType). - Namespace(c.ns). - Resource("mcads"). - Name(*name). - SubResource("status"). - VersionedParams(&patchOpts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} diff --git a/client/informer/externalversions/codeflare/interface.go b/client/informer/externalversions/codeflare/interface.go deleted file mode 100644 index 63ab10ff..00000000 --- a/client/informer/externalversions/codeflare/interface.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by informer-gen. DO NOT EDIT. - -package codeflare - -import ( - v1alpha1 "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/codeflare/v1alpha1" - internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" -) - -// Interface provides access to each of this group's versions. -type Interface interface { - // V1alpha1 provides access to shared informers for resources in V1alpha1. - V1alpha1() v1alpha1.Interface -} - -type group struct { - factory internalinterfaces.SharedInformerFactory - namespace string - tweakListOptions internalinterfaces.TweakListOptionsFunc -} - -// New returns a new Interface. -func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { - return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} -} - -// V1alpha1 returns a new v1alpha1.Interface. -func (g *group) V1alpha1() v1alpha1.Interface { - return v1alpha1.New(g.factory, g.namespace, g.tweakListOptions) -} diff --git a/client/informer/externalversions/codeflare/v1alpha1/instascale.go b/client/informer/externalversions/codeflare/v1alpha1/instascale.go deleted file mode 100644 index 79d6c7ac..00000000 --- a/client/informer/externalversions/codeflare/v1alpha1/instascale.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by informer-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "context" - time "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - watch "k8s.io/apimachinery/pkg/watch" - cache "k8s.io/client-go/tools/cache" - - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" - v1alpha1 "github.com/project-codeflare/codeflare-operator/client/listers/codeflare/v1alpha1" -) - -// InstaScaleInformer provides access to a shared informer and lister for -// InstaScales. -type InstaScaleInformer interface { - Informer() cache.SharedIndexInformer - Lister() v1alpha1.InstaScaleLister -} - -type instaScaleInformer struct { - factory internalinterfaces.SharedInformerFactory - tweakListOptions internalinterfaces.TweakListOptionsFunc - namespace string -} - -// NewInstaScaleInformer constructs a new informer for InstaScale type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewInstaScaleInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { - return NewFilteredInstaScaleInformer(client, namespace, resyncPeriod, indexers, nil) -} - -// NewFilteredInstaScaleInformer constructs a new informer for InstaScale type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewFilteredInstaScaleInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { - return cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options v1.ListOptions) (runtime.Object, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.CodeflareV1alpha1().InstaScales(namespace).List(context.TODO(), options) - }, - WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.CodeflareV1alpha1().InstaScales(namespace).Watch(context.TODO(), options) - }, - }, - &codeflarev1alpha1.InstaScale{}, - resyncPeriod, - indexers, - ) -} - -func (f *instaScaleInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - return NewFilteredInstaScaleInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) -} - -func (f *instaScaleInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&codeflarev1alpha1.InstaScale{}, f.defaultInformer) -} - -func (f *instaScaleInformer) Lister() v1alpha1.InstaScaleLister { - return v1alpha1.NewInstaScaleLister(f.Informer().GetIndexer()) -} diff --git a/client/informer/externalversions/codeflare/v1alpha1/interface.go b/client/informer/externalversions/codeflare/v1alpha1/interface.go deleted file mode 100644 index 6003a703..00000000 --- a/client/informer/externalversions/codeflare/v1alpha1/interface.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by informer-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" -) - -// Interface provides access to all the informers in this group version. -type Interface interface { - // InstaScales returns a InstaScaleInformer. - InstaScales() InstaScaleInformer - // MCADs returns a MCADInformer. - MCADs() MCADInformer -} - -type version struct { - factory internalinterfaces.SharedInformerFactory - namespace string - tweakListOptions internalinterfaces.TweakListOptionsFunc -} - -// New returns a new Interface. -func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { - return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} -} - -// InstaScales returns a InstaScaleInformer. -func (v *version) InstaScales() InstaScaleInformer { - return &instaScaleInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} -} - -// MCADs returns a MCADInformer. -func (v *version) MCADs() MCADInformer { - return &mCADInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} -} diff --git a/client/informer/externalversions/codeflare/v1alpha1/mcad.go b/client/informer/externalversions/codeflare/v1alpha1/mcad.go deleted file mode 100644 index 4cda692b..00000000 --- a/client/informer/externalversions/codeflare/v1alpha1/mcad.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by informer-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "context" - time "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - watch "k8s.io/apimachinery/pkg/watch" - cache "k8s.io/client-go/tools/cache" - - codeflarev1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" - versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" - v1alpha1 "github.com/project-codeflare/codeflare-operator/client/listers/codeflare/v1alpha1" -) - -// MCADInformer provides access to a shared informer and lister for -// MCADs. -type MCADInformer interface { - Informer() cache.SharedIndexInformer - Lister() v1alpha1.MCADLister -} - -type mCADInformer struct { - factory internalinterfaces.SharedInformerFactory - tweakListOptions internalinterfaces.TweakListOptionsFunc - namespace string -} - -// NewMCADInformer constructs a new informer for MCAD type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewMCADInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { - return NewFilteredMCADInformer(client, namespace, resyncPeriod, indexers, nil) -} - -// NewFilteredMCADInformer constructs a new informer for MCAD type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewFilteredMCADInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { - return cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options v1.ListOptions) (runtime.Object, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.CodeflareV1alpha1().MCADs(namespace).List(context.TODO(), options) - }, - WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.CodeflareV1alpha1().MCADs(namespace).Watch(context.TODO(), options) - }, - }, - &codeflarev1alpha1.MCAD{}, - resyncPeriod, - indexers, - ) -} - -func (f *mCADInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - return NewFilteredMCADInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) -} - -func (f *mCADInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&codeflarev1alpha1.MCAD{}, f.defaultInformer) -} - -func (f *mCADInformer) Lister() v1alpha1.MCADLister { - return v1alpha1.NewMCADLister(f.Informer().GetIndexer()) -} diff --git a/client/informer/externalversions/factory.go b/client/informer/externalversions/factory.go deleted file mode 100644 index 0d3733cb..00000000 --- a/client/informer/externalversions/factory.go +++ /dev/null @@ -1,252 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by informer-gen. DO NOT EDIT. - -package externalversions - -import ( - reflect "reflect" - sync "sync" - time "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - schema "k8s.io/apimachinery/pkg/runtime/schema" - cache "k8s.io/client-go/tools/cache" - - versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" - codeflare "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/codeflare" - internalinterfaces "github.com/project-codeflare/codeflare-operator/client/informer/externalversions/internalinterfaces" -) - -// SharedInformerOption defines the functional option type for SharedInformerFactory. -type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory - -type sharedInformerFactory struct { - client versioned.Interface - namespace string - tweakListOptions internalinterfaces.TweakListOptionsFunc - lock sync.Mutex - defaultResync time.Duration - customResync map[reflect.Type]time.Duration - - informers map[reflect.Type]cache.SharedIndexInformer - // startedInformers is used for tracking which informers have been started. - // This allows Start() to be called multiple times safely. - startedInformers map[reflect.Type]bool - // wg tracks how many goroutines were started. - wg sync.WaitGroup - // shuttingDown is true when Shutdown has been called. It may still be running - // because it needs to wait for goroutines. - shuttingDown bool -} - -// WithCustomResyncConfig sets a custom resync period for the specified informer types. -func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { - return func(factory *sharedInformerFactory) *sharedInformerFactory { - for k, v := range resyncConfig { - factory.customResync[reflect.TypeOf(k)] = v - } - return factory - } -} - -// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. -func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { - return func(factory *sharedInformerFactory) *sharedInformerFactory { - factory.tweakListOptions = tweakListOptions - return factory - } -} - -// WithNamespace limits the SharedInformerFactory to the specified namespace. -func WithNamespace(namespace string) SharedInformerOption { - return func(factory *sharedInformerFactory) *sharedInformerFactory { - factory.namespace = namespace - return factory - } -} - -// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. -func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { - return NewSharedInformerFactoryWithOptions(client, defaultResync) -} - -// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. -// Listers obtained via this SharedInformerFactory will be subject to the same filters -// as specified here. -// Deprecated: Please use NewSharedInformerFactoryWithOptions instead -func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { - return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) -} - -// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. -func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { - factory := &sharedInformerFactory{ - client: client, - namespace: v1.NamespaceAll, - defaultResync: defaultResync, - informers: make(map[reflect.Type]cache.SharedIndexInformer), - startedInformers: make(map[reflect.Type]bool), - customResync: make(map[reflect.Type]time.Duration), - } - - // Apply all options - for _, opt := range options { - factory = opt(factory) - } - - return factory -} - -func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { - f.lock.Lock() - defer f.lock.Unlock() - - if f.shuttingDown { - return - } - - for informerType, informer := range f.informers { - if !f.startedInformers[informerType] { - f.wg.Add(1) - // We need a new variable in each loop iteration, - // otherwise the goroutine would use the loop variable - // and that keeps changing. - informer := informer - go func() { - defer f.wg.Done() - informer.Run(stopCh) - }() - f.startedInformers[informerType] = true - } - } -} - -func (f *sharedInformerFactory) Shutdown() { - f.lock.Lock() - f.shuttingDown = true - f.lock.Unlock() - - // Will return immediately if there is nothing to wait for. - f.wg.Wait() -} - -func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { - informers := func() map[reflect.Type]cache.SharedIndexInformer { - f.lock.Lock() - defer f.lock.Unlock() - - informers := map[reflect.Type]cache.SharedIndexInformer{} - for informerType, informer := range f.informers { - if f.startedInformers[informerType] { - informers[informerType] = informer - } - } - return informers - }() - - res := map[reflect.Type]bool{} - for informType, informer := range informers { - res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) - } - return res -} - -// InternalInformerFor returns the SharedIndexInformer for obj using an internal -// client. -func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { - f.lock.Lock() - defer f.lock.Unlock() - - informerType := reflect.TypeOf(obj) - informer, exists := f.informers[informerType] - if exists { - return informer - } - - resyncPeriod, exists := f.customResync[informerType] - if !exists { - resyncPeriod = f.defaultResync - } - - informer = newFunc(f.client, resyncPeriod) - f.informers[informerType] = informer - - return informer -} - -// SharedInformerFactory provides shared informers for resources in all known -// API group versions. -// -// It is typically used like this: -// -// ctx, cancel := context.Background() -// defer cancel() -// factory := NewSharedInformerFactory(client, resyncPeriod) -// defer factory.WaitForStop() // Returns immediately if nothing was started. -// genericInformer := factory.ForResource(resource) -// typedInformer := factory.SomeAPIGroup().V1().SomeType() -// factory.Start(ctx.Done()) // Start processing these informers. -// synced := factory.WaitForCacheSync(ctx.Done()) -// for v, ok := range synced { -// if !ok { -// fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) -// return -// } -// } -// -// // Creating informers can also be created after Start, but then -// // Start must be called again: -// anotherGenericInformer := factory.ForResource(resource) -// factory.Start(ctx.Done()) -type SharedInformerFactory interface { - internalinterfaces.SharedInformerFactory - - // Start initializes all requested informers. They are handled in goroutines - // which run until the stop channel gets closed. - Start(stopCh <-chan struct{}) - - // Shutdown marks a factory as shutting down. At that point no new - // informers can be started anymore and Start will return without - // doing anything. - // - // In addition, Shutdown blocks until all goroutines have terminated. For that - // to happen, the close channel(s) that they were started with must be closed, - // either before Shutdown gets called or while it is waiting. - // - // Shutdown may be called multiple times, even concurrently. All such calls will - // block until all goroutines have terminated. - Shutdown() - - // WaitForCacheSync blocks until all started informers' caches were synced - // or the stop channel gets closed. - WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool - - // ForResource gives generic access to a shared informer of the matching type. - ForResource(resource schema.GroupVersionResource) (GenericInformer, error) - - // InternalInformerFor returns the SharedIndexInformer for obj using an internal - // client. - InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer - - Codeflare() codeflare.Interface -} - -func (f *sharedInformerFactory) Codeflare() codeflare.Interface { - return codeflare.New(f, f.namespace, f.tweakListOptions) -} diff --git a/client/informer/externalversions/generic.go b/client/informer/externalversions/generic.go deleted file mode 100644 index 6fa936ba..00000000 --- a/client/informer/externalversions/generic.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by informer-gen. DO NOT EDIT. - -package externalversions - -import ( - "fmt" - - schema "k8s.io/apimachinery/pkg/runtime/schema" - cache "k8s.io/client-go/tools/cache" - - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -// GenericInformer is type of SharedIndexInformer which will locate and delegate to other -// sharedInformers based on type -type GenericInformer interface { - Informer() cache.SharedIndexInformer - Lister() cache.GenericLister -} - -type genericInformer struct { - informer cache.SharedIndexInformer - resource schema.GroupResource -} - -// Informer returns the SharedIndexInformer. -func (f *genericInformer) Informer() cache.SharedIndexInformer { - return f.informer -} - -// Lister returns the GenericLister. -func (f *genericInformer) Lister() cache.GenericLister { - return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) -} - -// ForResource gives generic access to a shared informer of the matching type -// TODO extend this to unknown resources with a client pool -func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { - switch resource { - // Group=codeflare.codeflare.dev, Version=v1alpha1 - case v1alpha1.SchemeGroupVersion.WithResource("instascales"): - return &genericInformer{resource: resource.GroupResource(), informer: f.Codeflare().V1alpha1().InstaScales().Informer()}, nil - case v1alpha1.SchemeGroupVersion.WithResource("mcads"): - return &genericInformer{resource: resource.GroupResource(), informer: f.Codeflare().V1alpha1().MCADs().Informer()}, nil - - } - - return nil, fmt.Errorf("no informer found for %v", resource) -} diff --git a/client/informer/externalversions/internalinterfaces/factory_interfaces.go b/client/informer/externalversions/internalinterfaces/factory_interfaces.go deleted file mode 100644 index 0e89555b..00000000 --- a/client/informer/externalversions/internalinterfaces/factory_interfaces.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by informer-gen. DO NOT EDIT. - -package internalinterfaces - -import ( - time "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - cache "k8s.io/client-go/tools/cache" - - versioned "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" -) - -// NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. -type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer - -// SharedInformerFactory a small interface to allow for adding an informer without an import cycle -type SharedInformerFactory interface { - Start(stopCh <-chan struct{}) - InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer -} - -// TweakListOptionsFunc is a function that transforms a v1.ListOptions. -type TweakListOptionsFunc func(*v1.ListOptions) diff --git a/client/listers/codeflare/v1alpha1/expansion_generated.go b/client/listers/codeflare/v1alpha1/expansion_generated.go deleted file mode 100644 index 23082ab9..00000000 --- a/client/listers/codeflare/v1alpha1/expansion_generated.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by lister-gen. DO NOT EDIT. - -package v1alpha1 - -// InstaScaleListerExpansion allows custom methods to be added to -// InstaScaleLister. -type InstaScaleListerExpansion interface{} - -// InstaScaleNamespaceListerExpansion allows custom methods to be added to -// InstaScaleNamespaceLister. -type InstaScaleNamespaceListerExpansion interface{} - -// MCADListerExpansion allows custom methods to be added to -// MCADLister. -type MCADListerExpansion interface{} - -// MCADNamespaceListerExpansion allows custom methods to be added to -// MCADNamespaceLister. -type MCADNamespaceListerExpansion interface{} diff --git a/client/listers/codeflare/v1alpha1/instascale.go b/client/listers/codeflare/v1alpha1/instascale.go deleted file mode 100644 index c2519af5..00000000 --- a/client/listers/codeflare/v1alpha1/instascale.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by lister-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/tools/cache" - - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -// InstaScaleLister helps list InstaScales. -// All objects returned here must be treated as read-only. -type InstaScaleLister interface { - // List lists all InstaScales in the indexer. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1alpha1.InstaScale, err error) - // InstaScales returns an object that can list and get InstaScales. - InstaScales(namespace string) InstaScaleNamespaceLister - InstaScaleListerExpansion -} - -// instaScaleLister implements the InstaScaleLister interface. -type instaScaleLister struct { - indexer cache.Indexer -} - -// NewInstaScaleLister returns a new InstaScaleLister. -func NewInstaScaleLister(indexer cache.Indexer) InstaScaleLister { - return &instaScaleLister{indexer: indexer} -} - -// List lists all InstaScales in the indexer. -func (s *instaScaleLister) List(selector labels.Selector) (ret []*v1alpha1.InstaScale, err error) { - err = cache.ListAll(s.indexer, selector, func(m interface{}) { - ret = append(ret, m.(*v1alpha1.InstaScale)) - }) - return ret, err -} - -// InstaScales returns an object that can list and get InstaScales. -func (s *instaScaleLister) InstaScales(namespace string) InstaScaleNamespaceLister { - return instaScaleNamespaceLister{indexer: s.indexer, namespace: namespace} -} - -// InstaScaleNamespaceLister helps list and get InstaScales. -// All objects returned here must be treated as read-only. -type InstaScaleNamespaceLister interface { - // List lists all InstaScales in the indexer for a given namespace. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1alpha1.InstaScale, err error) - // Get retrieves the InstaScale from the indexer for a given namespace and name. - // Objects returned here must be treated as read-only. - Get(name string) (*v1alpha1.InstaScale, error) - InstaScaleNamespaceListerExpansion -} - -// instaScaleNamespaceLister implements the InstaScaleNamespaceLister -// interface. -type instaScaleNamespaceLister struct { - indexer cache.Indexer - namespace string -} - -// List lists all InstaScales in the indexer for a given namespace. -func (s instaScaleNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.InstaScale, err error) { - err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { - ret = append(ret, m.(*v1alpha1.InstaScale)) - }) - return ret, err -} - -// Get retrieves the InstaScale from the indexer for a given namespace and name. -func (s instaScaleNamespaceLister) Get(name string) (*v1alpha1.InstaScale, error) { - obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) - if err != nil { - return nil, err - } - if !exists { - return nil, errors.NewNotFound(v1alpha1.Resource("instascale"), name) - } - return obj.(*v1alpha1.InstaScale), nil -} diff --git a/client/listers/codeflare/v1alpha1/mcad.go b/client/listers/codeflare/v1alpha1/mcad.go deleted file mode 100644 index e2f47832..00000000 --- a/client/listers/codeflare/v1alpha1/mcad.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by lister-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/tools/cache" - - v1alpha1 "github.com/project-codeflare/codeflare-operator/api/codeflare/v1alpha1" -) - -// MCADLister helps list MCADs. -// All objects returned here must be treated as read-only. -type MCADLister interface { - // List lists all MCADs in the indexer. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1alpha1.MCAD, err error) - // MCADs returns an object that can list and get MCADs. - MCADs(namespace string) MCADNamespaceLister - MCADListerExpansion -} - -// mCADLister implements the MCADLister interface. -type mCADLister struct { - indexer cache.Indexer -} - -// NewMCADLister returns a new MCADLister. -func NewMCADLister(indexer cache.Indexer) MCADLister { - return &mCADLister{indexer: indexer} -} - -// List lists all MCADs in the indexer. -func (s *mCADLister) List(selector labels.Selector) (ret []*v1alpha1.MCAD, err error) { - err = cache.ListAll(s.indexer, selector, func(m interface{}) { - ret = append(ret, m.(*v1alpha1.MCAD)) - }) - return ret, err -} - -// MCADs returns an object that can list and get MCADs. -func (s *mCADLister) MCADs(namespace string) MCADNamespaceLister { - return mCADNamespaceLister{indexer: s.indexer, namespace: namespace} -} - -// MCADNamespaceLister helps list and get MCADs. -// All objects returned here must be treated as read-only. -type MCADNamespaceLister interface { - // List lists all MCADs in the indexer for a given namespace. - // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1alpha1.MCAD, err error) - // Get retrieves the MCAD from the indexer for a given namespace and name. - // Objects returned here must be treated as read-only. - Get(name string) (*v1alpha1.MCAD, error) - MCADNamespaceListerExpansion -} - -// mCADNamespaceLister implements the MCADNamespaceLister -// interface. -type mCADNamespaceLister struct { - indexer cache.Indexer - namespace string -} - -// List lists all MCADs in the indexer for a given namespace. -func (s mCADNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.MCAD, err error) { - err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { - ret = append(ret, m.(*v1alpha1.MCAD)) - }) - return ret, err -} - -// Get retrieves the MCAD from the indexer for a given namespace and name. -func (s mCADNamespaceLister) Get(name string) (*v1alpha1.MCAD, error) { - obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) - if err != nil { - return nil, err - } - if !exists { - return nil, errors.NewNotFound(v1alpha1.Resource("mcad"), name) - } - return obj.(*v1alpha1.MCAD), nil -} diff --git a/test/support/client.go b/test/support/client.go index 1a02e7eb..f5696bde 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -27,8 +27,6 @@ import ( imagev1 "github.com/openshift/client-go/image/clientset/versioned" routev1 "github.com/openshift/client-go/route/clientset/versioned" - - codeflareclient "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. ) @@ -37,20 +35,18 @@ type Client interface { Core() kubernetes.Interface Route() routev1.Interface Image() imagev1.Interface - CodeFlare() codeflareclient.Interface MCAD() mcadclient.Interface Ray() rayclient.Interface Dynamic() dynamic.Interface } type testClient struct { - core kubernetes.Interface - route routev1.Interface - image imagev1.Interface - codeflare codeflareclient.Interface - mcad mcadclient.Interface - ray rayclient.Interface - dynamic dynamic.Interface + core kubernetes.Interface + route routev1.Interface + image imagev1.Interface + mcad mcadclient.Interface + ray rayclient.Interface + dynamic dynamic.Interface } var _ Client = (*testClient)(nil) @@ -66,11 +62,6 @@ func (t *testClient) Route() routev1.Interface { func (t *testClient) Image() imagev1.Interface { return t.image } - -func (t *testClient) CodeFlare() codeflareclient.Interface { - return t.codeflare -} - func (t *testClient) MCAD() mcadclient.Interface { return t.mcad } @@ -107,11 +98,6 @@ func newTestClient() (Client, error) { return nil, err } - codeFlareClient, err := codeflareclient.NewForConfig(cfg) - if err != nil { - return nil, err - } - mcadClient, err := mcadclient.NewForConfig(cfg) if err != nil { return nil, err @@ -128,12 +114,11 @@ func newTestClient() (Client, error) { } return &testClient{ - core: kubeClient, - route: routeClient, - image: imageClient, - codeflare: codeFlareClient, - mcad: mcadClient, - ray: rayClient, - dynamic: dynamicClient, + core: kubeClient, + route: routeClient, + image: imageClient, + mcad: mcadClient, + ray: rayClient, + dynamic: dynamicClient, }, nil } From 1e560d9ba2c865080c17ef3dfaab344959ee58db Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 11 Aug 2023 18:14:37 +0200 Subject: [PATCH 121/377] Update go.mod --- go.mod | 10 +- go.sum | 287 --------------------------------------------------------- 2 files changed, 2 insertions(+), 295 deletions(-) diff --git a/go.mod b/go.mod index 75905947..3cacfb8f 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,6 @@ module github.com/project-codeflare/codeflare-operator go 1.19 require ( - github.com/go-logr/logr v1.2.4 - github.com/manifestival/controller-runtime-client v0.4.0 - github.com/manifestival/manifestival v0.7.2 - github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 @@ -19,7 +15,6 @@ require ( k8s.io/apimachinery v0.26.3 k8s.io/client-go v0.26.3 sigs.k8s.io/controller-runtime v0.14.6 - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 ) replace ( @@ -44,12 +39,12 @@ require ( github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.1 // indirect github.com/golang/glog v1.0.0 // indirect @@ -59,7 +54,6 @@ require ( github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect @@ -119,7 +113,6 @@ require ( golang.org/x/term v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.9.3 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect @@ -142,5 +135,6 @@ require ( sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 // indirect sigs.k8s.io/custom-metrics-apiserver v0.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index a95f13de..60247273 100644 --- a/go.sum +++ b/go.sum @@ -6,7 +6,6 @@ cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxK cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= @@ -34,57 +33,26 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Bobbins228/instascale v0.0.0-20230724130835-2a8ebf58690f h1:aEtM6OZ2dCY97VnVsVYZl2GJOe+6WegPGLgLXQt5BGk= github.com/Bobbins228/instascale v0.0.0-20230724130835-2a8ebf58690f/go.mod h1:kwBkC+o4W+a1dpftXbVQjvGCVyu2206HlZtcp1+ZjyY= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -95,7 +63,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -119,44 +86,28 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -168,32 +119,21 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.0.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/evanphx/json-patch/v5 v5.2.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -206,77 +146,27 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= -github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= -github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= -github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= -github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= -github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= @@ -284,7 +174,6 @@ github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -298,7 +187,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -317,11 +205,9 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/cel-go v0.12.6 h1:kjeKudqV0OygrAqA9fX6J55S8gj+Jre2tckIm5RoG4M= github.com/google/cel-go v0.12.6/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= @@ -333,15 +219,12 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -355,38 +238,23 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.4.0/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= @@ -409,7 +277,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -419,11 +286,8 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= @@ -484,10 +348,7 @@ github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9q github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -499,7 +360,6 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -512,7 +372,6 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -522,36 +381,21 @@ github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQSeYtvkY= github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/manifestival/controller-runtime-client v0.4.0 h1:AK+nTZ1rhPTfAc3upnSe3TiOvAWuRmxi1vjEr55dSUM= -github.com/manifestival/controller-runtime-client v0.4.0/go.mod h1:DrsviSegfdlpVsynTUZonKE5gFkMQupenlzVmDbC6S0= -github.com/manifestival/manifestival v0.6.0/go.mod h1:3Qq9cnPy7sv7FVhg2Kvw0ebb35R4OdctS4UjTjHlHm4= -github.com/manifestival/manifestival v0.7.2 h1:l4uFdWX/xQK4QcRfqGoMtBvaZeWPEuwD6hVsCwUqZY4= -github.com/manifestival/manifestival v0.7.2/go.mod h1:nl3T6HlfHCeidooWVTMI9vYNTBkQ1GdhLNb+smozbdk= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= @@ -568,8 +412,6 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -577,36 +419,23 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= @@ -620,11 +449,9 @@ github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHM github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -633,7 +460,6 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230904071404-ce734452edc6 h1:YT6C0Z84TgftYcoNUd/dwvXxGJvXy67EeWAMQjpKvdU= github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230904071404-ce734452edc6/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -676,15 +502,12 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -698,25 +521,19 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ= github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= @@ -730,7 +547,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -738,19 +554,13 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -759,11 +569,7 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5 h1:Gqga3zA9tdAcfqobUGjSoCob5L3f8Dt5EuOp3ihNZko= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= @@ -774,9 +580,6 @@ go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMx go.etcd.io/etcd/pkg/v3 v3.5.5 h1:Ablg7T7OkR+AeeeU32kdVhw/AGDsitkKPl7aW73ssjU= go.etcd.io/etcd/raft/v3 v3.5.5 h1:Ibz6XyZ60OYyRopu73lLM/P+qco3YtlZMOhnXNS051I= go.etcd.io/etcd/server/v3 v3.5.5 h1:jNjYm/9s+f9A9r6+SC4RvNaz6AqixpOvhrFdT0PvIj0= -go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -817,32 +620,23 @@ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKY go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -884,11 +678,9 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -896,23 +688,18 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200219183655-46282727080f/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -926,7 +713,6 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= @@ -935,7 +721,6 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -943,7 +728,6 @@ golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -965,7 +749,6 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -976,22 +759,18 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1009,14 +788,11 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1027,7 +803,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1039,15 +814,12 @@ golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -1056,21 +828,14 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1081,10 +846,7 @@ golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1116,7 +878,6 @@ golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWc golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -1127,14 +888,12 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -1191,7 +950,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= @@ -1199,7 +957,6 @@ google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxv google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -1239,14 +996,11 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -1255,7 +1009,6 @@ gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= @@ -1273,8 +1026,6 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1282,81 +1033,43 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.15.7/go.mod h1:a/tUxscL+UxvYyA7Tj5DRc8ivYqJIO1Y5KDdlI6wSvo= -k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= -k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= k8s.io/api v0.26.3 h1:emf74GIQMTik01Aum9dPP0gAypL8JTLl/lHa4V9RFSU= k8s.io/api v0.26.3/go.mod h1:PXsqwPMXBSBcL1lJ9CYDKy7kIReUydukS5JiRlxC3qE= -k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI= k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM= -k8s.io/apimachinery v0.15.7/go.mod h1:Xc10RHc1U+F/e9GCloJ8QAeCGevSVP5xhOhqlE+e1kM= -k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.19.7/go.mod h1:6sRbGRAVY5DOCuZwB5XkqguBqpqLU6q/kOaOdk29z6Q= -k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= k8s.io/apimachinery v0.26.3 h1:dQx6PNETJ7nODU3XPtrwkfuubs6w7sX0M8n61zHIV/k= k8s.io/apimachinery v0.26.3/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= -k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= k8s.io/apiserver v0.26.2 h1:Pk8lmX4G14hYqJd1poHGC08G03nIHVqdJMR0SD3IH3o= k8s.io/apiserver v0.26.2/go.mod h1:GHcozwXgXsPuOJ28EnQ/jXEM9QeG6HT22YxSNmpYNh8= -k8s.io/client-go v0.15.7/go.mod h1:QMNB76d3lKPvPQdOOnnxUF693C3hnCzUbC2umg70pWA= -k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= -k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= k8s.io/client-go v0.26.3 h1:k1UY+KXfkxV2ScEL3gilKcF7761xkYsSD6BC9szIu8s= k8s.io/client-go v0.26.3/go.mod h1:ZPNu9lm8/dbRIPAgteN30RSXea6vrCpFvq+MateTUuQ= -k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= -k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= k8s.io/component-base v0.26.2 h1:IfWgCGUDzrD6wLLgXEstJKYZKAFS2kO+rBRi0p3LqcI= k8s.io/component-base v0.26.2/go.mod h1:DxbuIe9M3IZPRxPIzhch2m1eT7uFrSBJUBuVCQEBivs= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kms v0.26.2 h1:GM1gg3tFK3OUU/QQFi93yGjG3lJT8s8l3Wkn2+VxBLM= k8s.io/kms v0.26.2/go.mod h1:69qGnf1NsFOQP07fBYqNLZklqEHSJF024JqYCaeVxHg= -k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= -k8s.io/kube-openapi v0.0.0-20200204173128-addea2498afe/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d h1:VcFq5n7wCJB2FQMCIHfC+f+jNcGgNMar1uKd6rVlifU= k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= k8s.io/metrics v0.26.2 h1:2gUvUWWnHPdE2tyA5DvyHC8HGryr+izhY9i5dzLP06s= k8s.io/metrics v0.26.2/go.mod h1:PX1wm9REV9hSGuw9GcXTFNDgab1KRXck3mNeiLYbRho= -k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= -k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 h1:+xBL5uTc+BkPBwmMi3vYfUJjq+N3K+H6PXeETwf5cPI= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo= -sigs.k8s.io/controller-runtime v0.7.2/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 h1:puQ5YlyBjhxg+OQ1YPMJXwtk7WhC4E6AlWIQ9pC8jws= sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3/go.mod h1:9nUXR/EgdYZto1aQ6yhwOksPR7J979jSyOqic1IgaOo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 262a477a1a80cb5d26ae71f5edfde037e4323961 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 24 Aug 2023 11:03:12 +0200 Subject: [PATCH 122/377] Fix OLM upgrade workflow --- .github/workflows/olm_tests.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 67c797be..405c74a9 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -115,8 +115,11 @@ jobs: CSV_VERSION=$(kubectl get ClusterServiceVersion -l operators.coreos.com/codeflare-operator.openshift-operators='' -n openshift-operators -o json | jq -r .items[].spec.version) echo "PREVIOUS_VERSION=v$CSV_VERSION" >> $GITHUB_ENV - - name: Deploy CodeFlare stack (MCAD, KubeRay) + - name: Deploy CodeFlare stack run: | + # FIXME + kubectl apply -n openshift-operators -f config/default/instascale-configmap.yaml + make setup-e2e - name: Build operator and catalog image @@ -177,12 +180,6 @@ jobs: echo "Printing CodeFlare operator logs" kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log - - name: Print MCAD controller logs - if: always() && steps.deploy.outcome == 'success' - run: | - echo "Printing MCAD controller logs" - kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log - - name: Print KubeRay operator logs if: always() && steps.deploy.outcome == 'success' run: | From 14b0b95bbe3b0e786f16b1375077665d50ba5507 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 24 Aug 2023 11:09:37 +0200 Subject: [PATCH 123/377] Remove MCAD and InstaScale internal templates --- Dockerfile | 1 - .../internal/instascale/clusterrole.yaml.tmpl | 73 ------------------- .../instascale/clusterrolebinding.yaml.tmpl | 16 ---- .../internal/instascale/configmap.yaml.tmpl | 11 --- .../internal/instascale/deployment.yaml.tmpl | 35 --------- config/internal/instascale/sa.yaml.tmpl | 5 -- .../clusterrole_mcad-controller.yaml.tmpl | 58 --------------- ...g_mcad-controller-kube-scheduler.yaml.tmpl | 19 ----- ...usterrolebinding_mcad-controller.yaml.tmpl | 19 ----- .../clusterrolebinding_mcad-edit.yaml.tmpl | 19 ----- config/internal/mcad/configmap.yaml.tmpl | 14 ---- config/internal/mcad/deployment.yaml.tmpl | 51 ------------- config/internal/mcad/image_secret.yaml | 10 --- config/internal/mcad/service_nodeport.yaml | 17 ----- config/internal/mcad/serviceaccount.yaml.tmpl | 5 -- main.go | 1 - 16 files changed, 354 deletions(-) delete mode 100644 config/internal/instascale/clusterrole.yaml.tmpl delete mode 100644 config/internal/instascale/clusterrolebinding.yaml.tmpl delete mode 100644 config/internal/instascale/configmap.yaml.tmpl delete mode 100644 config/internal/instascale/deployment.yaml.tmpl delete mode 100644 config/internal/instascale/sa.yaml.tmpl delete mode 100644 config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl delete mode 100644 config/internal/mcad/clusterrolebinding_mcad-controller-kube-scheduler.yaml.tmpl delete mode 100644 config/internal/mcad/clusterrolebinding_mcad-controller.yaml.tmpl delete mode 100644 config/internal/mcad/clusterrolebinding_mcad-edit.yaml.tmpl delete mode 100644 config/internal/mcad/configmap.yaml.tmpl delete mode 100644 config/internal/mcad/deployment.yaml.tmpl delete mode 100644 config/internal/mcad/image_secret.yaml delete mode 100644 config/internal/mcad/service_nodeport.yaml delete mode 100644 config/internal/mcad/serviceaccount.yaml.tmpl diff --git a/Dockerfile b/Dockerfile index f9614a70..4aaf9909 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,6 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7 WORKDIR / COPY --from=builder /workspace/manager . -COPY config/internal config/internal USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/config/internal/instascale/clusterrole.yaml.tmpl b/config/internal/instascale/clusterrole.yaml.tmpl deleted file mode 100644 index 90e60d37..00000000 --- a/config/internal/instascale/clusterrole.yaml.tmpl +++ /dev/null @@ -1,73 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: instascale-{{.Name}}-cr - labels: - app.kubernetes.io/managed-by: InstaScale - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} -rules: -- apiGroups: - - "" - resources: - - nodes - - configmaps - verbs: - - list - - watch - - get - - create - - update - - delete - - patch - -- apiGroups: - - "" - resourceNames: - - instascale-ocm-secret - resources: - - secrets - verbs: - - get - -- apiGroups: - - config.openshift.io - resources: - - clusterversions - verbs: - - get - - list - -- apiGroups: - - apps - resources: - - deployments - verbs: - - list - - watch - - get -- apiGroups: - - machine.openshift.io - resources: - - "*" - verbs: - - list - - watch - - get - - create - - update - - delete - - patch - -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers - verbs: - - list - - watch - - get - - create - - update - - delete - - patch diff --git a/config/internal/instascale/clusterrolebinding.yaml.tmpl b/config/internal/instascale/clusterrolebinding.yaml.tmpl deleted file mode 100644 index ef775b42..00000000 --- a/config/internal/instascale/clusterrolebinding.yaml.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: instascale-{{.Name}}-crb - labels: - app.kubernetes.io/managed-by: InstaScale - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: instascale-{{.Name}}-cr -subjects: -- kind: ServiceAccount - name: instascale-{{.Name}}-sa - namespace: {{.Namespace}} diff --git a/config/internal/instascale/configmap.yaml.tmpl b/config/internal/instascale/configmap.yaml.tmpl deleted file mode 100644 index 8e665ecc..00000000 --- a/config/internal/instascale/configmap.yaml.tmpl +++ /dev/null @@ -1,11 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: instascale-config - namespace: {{.Namespace}} - labels: - app: instascale-{{.Name}} - component: instascale -data: - maxScaleoutAllowed: "{{.MaxScaleoutAllowed}}" - useMachinePools: "{{.UseMachinePools}}" diff --git a/config/internal/instascale/deployment.yaml.tmpl b/config/internal/instascale/deployment.yaml.tmpl deleted file mode 100644 index fe21ade6..00000000 --- a/config/internal/instascale/deployment.yaml.tmpl +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: instascale-{{.Name}} - name: instascale-{{.Name}} - namespace: {{.Namespace}} -spec: - replicas: 1 - selector: - matchLabels: - app: instascale-{{.Name}} - template: - metadata: - labels: - app: instascale-{{.Name}} - spec: - containers: - - args: - - "--configs-namespace={{.Namespace}}" - image: {{.ControllerImage}} - name: instascale - resources: {{.ControllerResources}} - livenessProbe: - httpGet: - path: /healthz - port: 8081 - periodSeconds: 5 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - periodSeconds: 10 - serviceAccountName: instascale-{{.Name}}-sa diff --git a/config/internal/instascale/sa.yaml.tmpl b/config/internal/instascale/sa.yaml.tmpl deleted file mode 100644 index 562265ac..00000000 --- a/config/internal/instascale/sa.yaml.tmpl +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: instascale-{{.Name}}-sa - namespace: {{.Namespace}} diff --git a/config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl b/config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl deleted file mode 100644 index 8812cfc3..00000000 --- a/config/internal/mcad/clusterrole_mcad-controller.yaml.tmpl +++ /dev/null @@ -1,58 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: {{.Name}}-mcad-controller-role - labels: - kubernetes.io/bootstrapping: rbac-defaults - app.kubernetes.io/managed-by: MCAD - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} -rules: - - apiGroups: - - quota.codeflare.dev - - workload.codeflare.dev - resources: - - queuejobs - - schedulingspecs - - quotasubtrees - - appwrappers - - appwrappers/finalizers - - appwrappers/status - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - persistentvolumes - - namespaces - - lists - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch - - apiGroups: - - scheduling.sigs.k8s.io - resources: - - podgroups - verbs: - - get - - list - - watch - - create - - update - - patch - - delete diff --git a/config/internal/mcad/clusterrolebinding_mcad-controller-kube-scheduler.yaml.tmpl b/config/internal/mcad/clusterrolebinding_mcad-controller-kube-scheduler.yaml.tmpl deleted file mode 100644 index f01636b1..00000000 --- a/config/internal/mcad/clusterrolebinding_mcad-controller-kube-scheduler.yaml.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - kubernetes.io/bootstrapping: rbac-defaults - app.kubernetes.io/managed-by: MCAD - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} - name: {{.Name}}-mcad-controller-kube-scheduler-crb -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-scheduler -subjects: - - kind: ServiceAccount - name: mcad-controller-{{.Name}} - namespace: {{.Namespace}} diff --git a/config/internal/mcad/clusterrolebinding_mcad-controller.yaml.tmpl b/config/internal/mcad/clusterrolebinding_mcad-controller.yaml.tmpl deleted file mode 100644 index 6ef7ff52..00000000 --- a/config/internal/mcad/clusterrolebinding_mcad-controller.yaml.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - kubernetes.io/bootstrapping: rbac-defaults - app.kubernetes.io/managed-by: MCAD - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} - name: {{.Name}}-mcad-controller-crb -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{.Name}}-mcad-controller-role -subjects: - - kind: ServiceAccount - name: mcad-controller-{{.Name}} - namespace: {{.Namespace}} diff --git a/config/internal/mcad/clusterrolebinding_mcad-edit.yaml.tmpl b/config/internal/mcad/clusterrolebinding_mcad-edit.yaml.tmpl deleted file mode 100644 index 06b0f12d..00000000 --- a/config/internal/mcad/clusterrolebinding_mcad-edit.yaml.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - labels: - kubernetes.io/bootstrapping: rbac-defaults - app.kubernetes.io/managed-by: MCAD - codeflare.codeflare.dev/cr-name: {{.Name}} - codeflare.codeflare.dev/cr-namespace: {{.Namespace}} - name: {{.Name}}-mcad-edit-crb -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: edit -subjects: - - kind: ServiceAccount - name: mcad-controller-{{.Name}} - namespace: {{.Namespace}} diff --git a/config/internal/mcad/configmap.yaml.tmpl b/config/internal/mcad/configmap.yaml.tmpl deleted file mode 100644 index c99b4a70..00000000 --- a/config/internal/mcad/configmap.yaml.tmpl +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -data: - DISPATCHER_MODE: "{{.DispatcherMode}}" - PREEMPTION: "{{.PreemptionEnabled}}" - {{if ne .AgentConfigs ""}}DISPATCHER_AGENT_CONFIGS: "{{.AgentConfigs}}"{{end}} - {{if ne .QuotaRestURL ""}}QUOTA_REST_URL: "{{.QuotaRestURL}}"{{end}} - {{if ne .PodCreationTimeout -1}}DISPATCH_RESOURCE_RESERVATION_TIMEOUT: "{{.PodCreationTimeout}}"{{end}} -kind: ConfigMap -metadata: - name: mcad-{{.Name}}-config - namespace: {{.Namespace}} - labels: - app: mcad-{{.Name}} - component: multi-cluster-app-dispatcher diff --git a/config/internal/mcad/deployment.yaml.tmpl b/config/internal/mcad/deployment.yaml.tmpl deleted file mode 100644 index 502ab69d..00000000 --- a/config/internal/mcad/deployment.yaml.tmpl +++ /dev/null @@ -1,51 +0,0 @@ -kind: Deployment -apiVersion: apps/v1 -metadata: - name: mcad-controller-{{.Name}} - namespace: {{.Namespace}} - labels: - app: mcad-{{.Name}} - component: multi-cluster-application-dispatcher -spec: - replicas: 1 - selector: - matchLabels: - app: mcad-{{.Name}} - template: - metadata: - labels: - app: mcad-{{.Name}} - component: multi-cluster-application-dispatcher - spec: - containers: - - name: mcad-controller - args: ["--v", "4", "--logtostderr"] - command: - - mcad-controller - envFrom: - - configMapRef: - name: mcad-{{.Name}}-config - image: {{.ControllerImage}} - imagePullPolicy: Always - resources: {{.ControllerResources}} - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - name: temp-vol - mountPath: /tmp - livenessProbe: - httpGet: - path: /healthz - port: 8081 - timeoutSeconds: 5 - periodSeconds: 5 - readinessProbe: - httpGet: - path: /healthz - port: 8081 - timeoutSeconds: 5 - periodSeconds: 5 - serviceAccountName: mcad-controller-{{.Name}} - volumes: - - name: temp-vol - emptyDir: {} diff --git a/config/internal/mcad/image_secret.yaml b/config/internal/mcad/image_secret.yaml deleted file mode 100644 index 840c5069..00000000 --- a/config/internal/mcad/image_secret.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# {{ if .Values.imagePullSecret.name }} -apiVersion: v1 -kind: Secret -metadata: - name: {{.Values.imagePullSecret.name}} - namespace: kube-system -type: kubernetes.io/dockerconfigjson -data: - .dockerconfigjson: {{template "imagePullSecret" .}} -# {{ end }} diff --git a/config/internal/mcad/service_nodeport.yaml b/config/internal/mcad/service_nodeport.yaml deleted file mode 100644 index f474e840..00000000 --- a/config/internal/mcad/service_nodeport.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# {{ if .Values.configMap.quotaRestUrl }} -apiVersion: v1 -kind: Service -metadata: - name: mcad-{{.Name}}-quota-http-server -spec: - type: NodePort - selector: - app: mcad-{{.Name}} - ports: - # By default and for convenience, the `targetPort` is set to the same value as the `port` field. - - port: 8082 - targetPort: 80 - # Optional field - # By default and for convenience, the Kubernetes control plane will allocate a port from a range (default: 30000-32767) - nodePort: 30082 -# {{ end }} diff --git a/config/internal/mcad/serviceaccount.yaml.tmpl b/config/internal/mcad/serviceaccount.yaml.tmpl deleted file mode 100644 index c0418d24..00000000 --- a/config/internal/mcad/serviceaccount.yaml.tmpl +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: mcad-controller-{{.Name}} - namespace: {{.Namespace}} diff --git a/main.go b/main.go index 73288133..150547f5 100644 --- a/main.go +++ b/main.go @@ -42,7 +42,6 @@ import ( var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") - // templatesPath = "config/internal/" ) func init() { From d2daec11dc8dce5a9c72e78e8b214ff6df03a853 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 5 Sep 2023 09:51:08 +0200 Subject: [PATCH 124/377] Remove MCAD and InstaScale CRDs --- Makefile | 4 +- PROJECT | 19 --- .../codeflare.codeflare.dev_instascales.yaml | 126 --------------- .../bases/codeflare.codeflare.dev_mcads.yaml | 144 ------------------ config/crd/kustomization.yaml | 19 --- config/crd/kustomizeconfig.yaml | 19 --- .../patches/cainjection_in_instascales.yaml | 7 - config/crd/patches/cainjection_in_mcads.yaml | 7 - .../crd/patches/webhook_in_instascales.yaml | 16 -- config/crd/patches/webhook_in_mcads.yaml | 16 -- 10 files changed, 2 insertions(+), 375 deletions(-) delete mode 100644 config/crd/bases/codeflare.codeflare.dev_instascales.yaml delete mode 100644 config/crd/bases/codeflare.codeflare.dev_mcads.yaml delete mode 100644 config/crd/kustomizeconfig.yaml delete mode 100644 config/crd/patches/cainjection_in_instascales.yaml delete mode 100644 config/crd/patches/cainjection_in_mcads.yaml delete mode 100644 config/crd/patches/webhook_in_instascales.yaml delete mode 100644 config/crd/patches/webhook_in_mcads.yaml diff --git a/Makefile b/Makefile index 5e14740b..0bee7bbc 100644 --- a/Makefile +++ b/Makefile @@ -151,8 +151,8 @@ defaults: gofmt -w $(DEFAULTS_TEST_FILE) .PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases +manifests: controller-gen ## Generate RBAC objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. diff --git a/PROJECT b/PROJECT index 6ed09b51..57b3ef28 100644 --- a/PROJECT +++ b/PROJECT @@ -6,23 +6,4 @@ plugins: scorecard.sdk.operatorframework.io/v2: {} projectName: codeflare-operator repo: github.com/project-codeflare/codeflare-operator -resources: -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: codeflare.dev - group: codeflare - kind: MCAD - path: github.com/project-codeflare/codeflare-operator/api/v1alpha1 - version: v1alpha1 -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: codeflare.dev - group: codeflare - kind: InstaScale - path: github.com/project-codeflare/codeflare-operator/api/v1alpha1 - version: v1alpha1 version: "3" diff --git a/config/crd/bases/codeflare.codeflare.dev_instascales.yaml b/config/crd/bases/codeflare.codeflare.dev_instascales.yaml deleted file mode 100644 index 26a72b76..00000000 --- a/config/crd/bases/codeflare.codeflare.dev_instascales.yaml +++ /dev/null @@ -1,126 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: instascales.codeflare.codeflare.dev -spec: - group: codeflare.codeflare.dev - names: - kind: InstaScale - listKind: InstaScaleList - plural: instascales - singular: instascale - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: InstaScale is the Schema for the instascales API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: InstaScaleSpec defines the desired state of InstaScale - properties: - controllerImage: - description: The container image for the InstaScale controller deployment. - If specified, the provided container image must be compatible with - the running CodeFlare operator. Using an incompatible, or unrelated - container image, will result in an undefined behavior. A CodeFlare - operator upgrade will not upgrade the InstaScale controller, that'll - keep running this specified container image. If not specified, the - latest version compatible with the running CodeFlare operator is - used. A CodeFlare operator upgrade may upgrade the InstaScale controller - to a newer container image. - type: string - controllerResources: - description: controllerResources determines the container resources - for the InstaScale controller deployment - properties: - claims: - description: "Claims lists the names of resources, defined in - spec.resourceClaims, that are used by this container. \n This - is an alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable. It can only be set - for containers." - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name of one entry in pod.spec.resourceClaims - of the Pod where this field is used. It makes that resource - available inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources - allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - enableMonitoring: - default: true - description: enableMonitoring determines if monitoring artifacts are - deployed for the InstaScale instance. - type: boolean - maxScaleoutAllowed: - default: 15 - description: maxScaleoutAllowed determines the max number of machines - that can be scaled up by InstaScale - type: integer - useMachinePools: - default: false - description: useMachinePools determines whether InstaScale should - use MachineSets or MachinePools for scaling - type: boolean - type: object - status: - description: InstaScaleStatus defines the observed state of InstaScale - properties: - ready: - default: false - type: boolean - required: - - ready - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/codeflare.codeflare.dev_mcads.yaml b/config/crd/bases/codeflare.codeflare.dev_mcads.yaml deleted file mode 100644 index 8172df33..00000000 --- a/config/crd/bases/codeflare.codeflare.dev_mcads.yaml +++ /dev/null @@ -1,144 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: mcads.codeflare.codeflare.dev -spec: - group: codeflare.codeflare.dev - names: - kind: MCAD - listKind: MCADList - plural: mcads - singular: mcad - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: MCAD is the Schema for the mcads API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: MCADSpec defines the desired state of MCAD - properties: - agentConfigs: - description: AgentConfigs determine paths to agent config file:deploymentName - separated by commas(,). - type: string - controllerImage: - description: The container image for the MCAD controller deployment. - If specified, the provided container image must be compatible with - the running CodeFlare operator. Using an incompatible, or unrelated - container image, will result in an undefined behavior. A CodeFlare - operator upgrade will not upgrade the MCAD controller, that'll keep - running this specified container image. If not specified, the latest - version compatible with the running CodeFlare operator is used. - A CodeFlare operator upgrade may upgrade the MCAD controller to - a newer container image. - type: string - controllerResources: - description: ControllerResources defines the cpu and memory resource - requirements for the MCAD Controller - properties: - claims: - description: "Claims lists the names of resources, defined in - spec.resourceClaims, that are used by this container. \n This - is an alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable. It can only be set - for containers." - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name of one entry in pod.spec.resourceClaims - of the Pod where this field is used. It makes that resource - available inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources - allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - dispatcherMode: - default: false - description: DispatcherMode determines whether the MCAD Controller - should be launched in Dispatcher mode. - type: boolean - enableMonitoring: - default: true - description: EnableMonitoring determines if monitoring artifacts are - deployed for the MCAD instance. - type: boolean - multiCluster: - default: false - description: MultiCluster determines if MCAD will be routing traffic - to multiple clusters. - type: boolean - podCreationTimeout: - default: -1 - description: PodCreationTimeout determines timeout in milliseconds - for pods to be created after dispatching job. - type: integer - preemptionEnabled: - default: false - description: PreemptionEnabled determines if scheduled jobs can be - preempted for others - type: boolean - quotaRestURL: - description: QuotaRestURL determines URL for Rest quota management. - type: string - type: object - status: - description: MCADStatus defines the observed state of MCAD - properties: - ready: - description: Ready indicates whether the application is ready to serve - requests - type: boolean - required: - - ready - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 725db012..81d71c1e 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,24 +2,5 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- bases/codeflare.codeflare.dev_mcads.yaml -- bases/codeflare.codeflare.dev_instascales.yaml - mcad #+kubebuilder:scaffold:crdkustomizeresource - -patchesStrategicMerge: -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. -# patches here are for enabling the conversion webhook for each CRD -#- patches/webhook_in_mcads.yaml -#- patches/webhook_in_instascales.yaml -#+kubebuilder:scaffold:crdkustomizewebhookpatch - -# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. -# patches here are for enabling the CA injection for each CRD -#- patches/cainjection_in_mcads.yaml -#- patches/cainjection_in_instascales.yaml -#+kubebuilder:scaffold:crdkustomizecainjectionpatch - -# the following config is for teaching kustomize how to do kustomization for CRDs. -configurations: -- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml deleted file mode 100644 index ec5c150a..00000000 --- a/config/crd/kustomizeconfig.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# This file is for teaching kustomize how to substitute name and namespace reference in CRD -nameReference: -- kind: Service - version: v1 - fieldSpecs: - - kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/name - -namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false - -varReference: -- path: metadata/annotations diff --git a/config/crd/patches/cainjection_in_instascales.yaml b/config/crd/patches/cainjection_in_instascales.yaml deleted file mode 100644 index 12b00004..00000000 --- a/config/crd/patches/cainjection_in_instascales.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: instascales.codeflare.codeflare.dev diff --git a/config/crd/patches/cainjection_in_mcads.yaml b/config/crd/patches/cainjection_in_mcads.yaml deleted file mode 100644 index ddaad38d..00000000 --- a/config/crd/patches/cainjection_in_mcads.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: mcads.codeflare.codeflare.dev diff --git a/config/crd/patches/webhook_in_instascales.yaml b/config/crd/patches/webhook_in_instascales.yaml deleted file mode 100644 index 55d5cf2b..00000000 --- a/config/crd/patches/webhook_in_instascales.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: instascales.codeflare.codeflare.dev -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 diff --git a/config/crd/patches/webhook_in_mcads.yaml b/config/crd/patches/webhook_in_mcads.yaml deleted file mode 100644 index ca858acf..00000000 --- a/config/crd/patches/webhook_in_mcads.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: mcads.codeflare.codeflare.dev -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: webhook-service - path: /convert - conversionReviewVersions: - - v1 From 88077115cdf060b35c1eb93b12f43775e4148968 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 5 Sep 2023 10:01:07 +0200 Subject: [PATCH 125/377] Update OLM ClusterServiceVersion --- ...eflare-operator.clusterserviceversion.yaml | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml index 1a45a5a7..77295679 100644 --- a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml @@ -14,16 +14,21 @@ spec: apiservicedefinitions: {} customresourcedefinitions: owned: - - description: InstaScale is the Schema for the instascales API - displayName: InstaScale - kind: InstaScale - name: instascales.codeflare.codeflare.dev - version: v1alpha1 - - description: MCAD is the Schema for the mcads API - displayName: MCAD - kind: MCAD - name: mcads.codeflare.codeflare.dev - version: v1alpha1 + - description: AppWrapper is the Schema for the AppWrapper API + displayName: AppWrapper + kind: AppWrappers + name: appwrappers.workload.codeflare.dev + version: v1beta1 + - description: SchedulingSpec is the Schema for the SchedulingSpec API + displayName: SchedulingSpec + kind: SchedulingSpecs + name: schedulingspecs.workload.codeflare.dev + version: v1beta1 + - description: QuotaSubtree is the Schema for the QuotaSubtree API + displayName: QuotaSubtree + kind: QuotaSubtrees + name: quotasubtrees.quota.codeflare.dev + version: v1 description: CodeFlare allows you to scale complex pipelines anywhere displayName: CodeFlare Operator icon: From c8373ed4da0ff7c680a479f473ca9acc856966bc Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 8 Sep 2023 11:37:14 +0200 Subject: [PATCH 126/377] Remove MCAD and InstaScale APIs --- Dockerfile | 1 - Makefile | 12 +- api/codeflare/v1alpha1/doc.go | 21 -- api/codeflare/v1alpha1/instascale_types.go | 83 ------- api/codeflare/v1alpha1/mcad_types.go | 97 -------- api/codeflare/v1alpha1/register.go | 51 ----- .../v1alpha1/zz_generated.deepcopy.go | 211 ------------------ 7 files changed, 4 insertions(+), 472 deletions(-) delete mode 100644 api/codeflare/v1alpha1/doc.go delete mode 100644 api/codeflare/v1alpha1/instascale_types.go delete mode 100644 api/codeflare/v1alpha1/mcad_types.go delete mode 100644 api/codeflare/v1alpha1/register.go delete mode 100644 api/codeflare/v1alpha1/zz_generated.deepcopy.go diff --git a/Dockerfile b/Dockerfile index 4aaf9909..b896f03b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,6 @@ RUN go mod download # Copy the go source COPY main.go main.go -COPY api/ api/ # Build USER root diff --git a/Makefile b/Makefile index 0bee7bbc..3c0c2e13 100644 --- a/Makefile +++ b/Makefile @@ -154,10 +154,6 @@ defaults: manifests: controller-gen ## Generate RBAC objects. $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." -.PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./api/..." - .PHONY: fmt fmt: ## Run go fmt against code. go fmt ./... @@ -175,11 +171,11 @@ modules: ## Update Go dependencies. #go get github.com/ray-project/kuberay/ray-operator .PHONY: build -build: modules defaults generate fmt vet ## Build manager binary. +build: modules defaults fmt vet ## Build manager binary. go build -o bin/manager main.go .PHONY: run -run: modules defaults manifests generate fmt vet ## Run a controller from your host. +run: modules defaults manifests fmt vet ## Run a controller from your host. go run ./main.go .PHONY: image-build @@ -367,11 +363,11 @@ catalog-push: ## Push a catalog image. podman push $(CATALOG_IMG) $(CATALOG_PUSH_OPT) .PHONY: test-unit -test-unit: defaults manifests generate fmt vet envtest ## Run unit tests. +test-unit: defaults manifests fmt vet envtest ## Run unit tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(go list ./... | grep -v /test/) -coverprofile cover.out .PHONY: test-e2e -test-e2e: defaults manifests generate fmt vet ## Run e2e tests. +test-e2e: defaults manifests fmt vet ## Run e2e tests. go test -timeout 30m -v ./test/e2e .PHONY: kind-e2e diff --git a/api/codeflare/v1alpha1/doc.go b/api/codeflare/v1alpha1/doc.go deleted file mode 100644 index 13be1de4..00000000 --- a/api/codeflare/v1alpha1/doc.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// +k8s:deepcopy-gen=package,register -// +kubebuilder:object:generate=true -// +groupName=codeflare.codeflare.dev - -package v1alpha1 diff --git a/api/codeflare/v1alpha1/instascale_types.go b/api/codeflare/v1alpha1/instascale_types.go deleted file mode 100644 index 7368cb1a..00000000 --- a/api/codeflare/v1alpha1/instascale_types.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// InstaScaleSpec defines the desired state of InstaScale -type InstaScaleSpec struct { - - // enableMonitoring determines if monitoring artifacts are deployed for the InstaScale instance. - // +kubebuilder:default=true - EnableMonitoring bool `json:"enableMonitoring,omitempty"` - - // maxScaleoutAllowed determines the max number of machines that can be scaled up by InstaScale - // +kubebuilder:default=15 - MaxScaleoutAllowed int `json:"maxScaleoutAllowed,omitempty"` - - // useMachinePools determines whether InstaScale should use MachineSets or MachinePools for scaling - // +kubebuilder:default=false - UseMachinePools bool `json:"useMachinePools,omitempty"` - - // controllerResources determines the container resources for the InstaScale controller deployment - ControllerResources *v1.ResourceRequirements `json:"controllerResources,omitempty"` - - // The container image for the InstaScale controller deployment. - // If specified, the provided container image must be compatible with the running CodeFlare operator. - // Using an incompatible, or unrelated container image, will result in an undefined behavior. - // A CodeFlare operator upgrade will not upgrade the InstaScale controller, that'll keep running this - // specified container image. - // If not specified, the latest version compatible with the running CodeFlare operator is used. - // A CodeFlare operator upgrade may upgrade the InstaScale controller to a newer container image. - // - // +optional - ControllerImage string `json:"controllerImage,omitempty"` -} - -// InstaScaleStatus defines the observed state of InstaScale -type InstaScaleStatus struct { - // Important: Run "make" to regenerate code after modifying this file - - // +kubebuilder:default=false - Ready bool `json:"ready"` -} - -// +genclient -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status - -// InstaScale is the Schema for the instascales API -// +operator-sdk:csv:customresourcedefinitions:displayName="InstaScale" -type InstaScale struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec InstaScaleSpec `json:"spec,omitempty"` - Status InstaScaleStatus `json:"status,omitempty"` -} - -// +kubebuilder:object:root=true - -// InstaScaleList contains a list of InstaScale -type InstaScaleList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []InstaScale `json:"items"` -} diff --git a/api/codeflare/v1alpha1/mcad_types.go b/api/codeflare/v1alpha1/mcad_types.go deleted file mode 100644 index dc22291f..00000000 --- a/api/codeflare/v1alpha1/mcad_types.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// MCADSpec defines the desired state of MCAD -type MCADSpec struct { - // EnableMonitoring determines if monitoring artifacts are deployed for the MCAD instance. - // +kubebuilder:default=true - EnableMonitoring bool `json:"enableMonitoring,omitempty"` - - // MultiCluster determines if MCAD will be routing traffic to multiple clusters. - // +kubebuilder:default=false - MultiCluster bool `json:"multiCluster,omitempty"` - - // DispatcherMode determines whether the MCAD Controller should be launched in Dispatcher mode. - // +kubebuilder:default=false - DispatcherMode bool `json:"dispatcherMode,omitempty"` - - // PreemptionEnabled determines if scheduled jobs can be preempted for others - // +kubebuilder:default=false - PreemptionEnabled bool `json:"preemptionEnabled,omitempty"` - - // AgentConfigs determine paths to agent config file:deploymentName separated by commas(,). - AgentConfigs string `json:"agentConfigs,omitempty"` - - // QuotaRestURL determines URL for Rest quota management. - QuotaRestURL string `json:"quotaRestURL,omitempty"` - - // PodCreationTimeout determines timeout in milliseconds for pods to be created after dispatching job. - // +kubebuilder:default=-1 - PodCreationTimeout int `json:"podCreationTimeout,omitempty"` - // podCreationTimeout: //int (default blank) - - // ControllerResources defines the cpu and memory resource requirements for the MCAD Controller - // +kubebuilder:default={} - ControllerResources v1.ResourceRequirements `json:"controllerResources,omitempty" protobuf:"bytes,8,opt"` - - // The container image for the MCAD controller deployment. - // If specified, the provided container image must be compatible with the running CodeFlare operator. - // Using an incompatible, or unrelated container image, will result in an undefined behavior. - // A CodeFlare operator upgrade will not upgrade the MCAD controller, that'll keep running this - // specified container image. - // If not specified, the latest version compatible with the running CodeFlare operator is used. - // A CodeFlare operator upgrade may upgrade the MCAD controller to a newer container image. - // - // +optional - ControllerImage string `json:"controllerImage,omitempty"` -} - -// MCADStatus defines the observed state of MCAD -type MCADStatus struct { - // Important: Run "make" to regenerate code after modifying this file - - // Ready indicates whether the application is ready to serve requests - Ready bool `json:"ready"` -} - -// +genclient -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status - -// MCAD is the Schema for the mcads API -type MCAD struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec MCADSpec `json:"spec,omitempty"` - Status MCADStatus `json:"status,omitempty"` -} - -// +kubebuilder:object:root=true - -// MCADList contains a list of MCAD -type MCADList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []MCAD `json:"items"` -} diff --git a/api/codeflare/v1alpha1/register.go b/api/codeflare/v1alpha1/register.go deleted file mode 100644 index 12bd3997..00000000 --- a/api/codeflare/v1alpha1/register.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// Kind takes an unqualified kind and returns back a Group qualified GroupKind. -func Kind(kind string) schema.GroupKind { - return SchemeGroupVersion.WithKind(kind).GroupKind() -} - -// Resource takes an unqualified resource and returns a Group qualified GroupResource. -func Resource(resource string) schema.GroupResource { - return SchemeGroupVersion.WithResource(resource).GroupResource() -} - -var ( - SchemeGroupVersion = schema.GroupVersion{Group: "codeflare.codeflare.dev", Version: "v1alpha1"} - SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) - AddToScheme = SchemeBuilder.AddToScheme -) - -// Adds the list of known types to Scheme. -func addKnownTypes(scheme *runtime.Scheme) error { - scheme.AddKnownTypes(SchemeGroupVersion, - &MCAD{}, - &MCADList{}, - &InstaScale{}, - &InstaScaleList{}, - ) - metav1.AddToGroupVersion(scheme, SchemeGroupVersion) - return nil -} diff --git a/api/codeflare/v1alpha1/zz_generated.deepcopy.go b/api/codeflare/v1alpha1/zz_generated.deepcopy.go deleted file mode 100644 index 5f0dd24a..00000000 --- a/api/codeflare/v1alpha1/zz_generated.deepcopy.go +++ /dev/null @@ -1,211 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InstaScale) DeepCopyInto(out *InstaScale) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstaScale. -func (in *InstaScale) DeepCopy() *InstaScale { - if in == nil { - return nil - } - out := new(InstaScale) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *InstaScale) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InstaScaleList) DeepCopyInto(out *InstaScaleList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]InstaScale, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstaScaleList. -func (in *InstaScaleList) DeepCopy() *InstaScaleList { - if in == nil { - return nil - } - out := new(InstaScaleList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *InstaScaleList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InstaScaleSpec) DeepCopyInto(out *InstaScaleSpec) { - *out = *in - if in.ControllerResources != nil { - in, out := &in.ControllerResources, &out.ControllerResources - *out = new(v1.ResourceRequirements) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstaScaleSpec. -func (in *InstaScaleSpec) DeepCopy() *InstaScaleSpec { - if in == nil { - return nil - } - out := new(InstaScaleSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InstaScaleStatus) DeepCopyInto(out *InstaScaleStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstaScaleStatus. -func (in *InstaScaleStatus) DeepCopy() *InstaScaleStatus { - if in == nil { - return nil - } - out := new(InstaScaleStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MCAD) DeepCopyInto(out *MCAD) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MCAD. -func (in *MCAD) DeepCopy() *MCAD { - if in == nil { - return nil - } - out := new(MCAD) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *MCAD) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MCADList) DeepCopyInto(out *MCADList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]MCAD, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MCADList. -func (in *MCADList) DeepCopy() *MCADList { - if in == nil { - return nil - } - out := new(MCADList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *MCADList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MCADSpec) DeepCopyInto(out *MCADSpec) { - *out = *in - in.ControllerResources.DeepCopyInto(&out.ControllerResources) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MCADSpec. -func (in *MCADSpec) DeepCopy() *MCADSpec { - if in == nil { - return nil - } - out := new(MCADSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MCADStatus) DeepCopyInto(out *MCADStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MCADStatus. -func (in *MCADStatus) DeepCopy() *MCADStatus { - if in == nil { - return nil - } - out := new(MCADStatus) - in.DeepCopyInto(out) - return out -} From f5bc3c1e51dc5c4e2c1e77b39da6963499510211 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 8 Sep 2023 11:40:03 +0200 Subject: [PATCH 127/377] Remove MCAD and InstaScale APIs RBAC and samples --- config/manifests/kustomization.yaml | 1 - config/rbac/instascale_editor_role.yaml | 31 ----------- config/rbac/instascale_viewer_role.yaml | 27 ---------- config/rbac/mcad_editor_role.yaml | 24 --------- config/rbac/mcad_viewer_role.yaml | 20 ------- config/rbac/role.yaml | 52 ------------------- .../codeflare_v1alpha1_instascale.yaml | 12 ----- config/samples/codeflare_v1alpha1_mcad.yaml | 6 --- config/samples/kustomization.yaml | 5 -- 9 files changed, 178 deletions(-) delete mode 100644 config/rbac/instascale_editor_role.yaml delete mode 100644 config/rbac/instascale_viewer_role.yaml delete mode 100644 config/rbac/mcad_editor_role.yaml delete mode 100644 config/rbac/mcad_viewer_role.yaml delete mode 100644 config/samples/codeflare_v1alpha1_instascale.yaml delete mode 100644 config/samples/codeflare_v1alpha1_mcad.yaml delete mode 100644 config/samples/kustomization.yaml diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml index 954c255a..958a9e24 100644 --- a/config/manifests/kustomization.yaml +++ b/config/manifests/kustomization.yaml @@ -3,7 +3,6 @@ resources: - bases/codeflare-operator.clusterserviceversion.yaml - ../default -- ../samples - ../scorecard apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/config/rbac/instascale_editor_role.yaml b/config/rbac/instascale_editor_role.yaml deleted file mode 100644 index 47530295..00000000 --- a/config/rbac/instascale_editor_role.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# permissions for end users to edit instascales. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: instascale-editor-role - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: codeflare-operator - app.kubernetes.io/part-of: codeflare-operator - app.kubernetes.io/managed-by: kustomize - name: instascale-editor-role -rules: -- apiGroups: - - codeflare.codeflare.dev - resources: - - instascales - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - codeflare.codeflare.dev - resources: - - instascales/status - verbs: - - get diff --git a/config/rbac/instascale_viewer_role.yaml b/config/rbac/instascale_viewer_role.yaml deleted file mode 100644 index ca4fc68e..00000000 --- a/config/rbac/instascale_viewer_role.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# permissions for end users to view instascales. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: instascale-viewer-role - app.kubernetes.io/component: rbac - app.kubernetes.io/created-by: codeflare-operator - app.kubernetes.io/part-of: codeflare-operator - app.kubernetes.io/managed-by: kustomize - name: instascale-viewer-role -rules: -- apiGroups: - - codeflare.codeflare.dev - resources: - - instascales - verbs: - - get - - list - - watch -- apiGroups: - - codeflare.codeflare.dev - resources: - - instascales/status - verbs: - - get diff --git a/config/rbac/mcad_editor_role.yaml b/config/rbac/mcad_editor_role.yaml deleted file mode 100644 index f82b9507..00000000 --- a/config/rbac/mcad_editor_role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# permissions for end users to edit mcads. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: mcad-editor-role -rules: -- apiGroups: - - codeflare.codeflare.dev - resources: - - mcads - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - codeflare.codeflare.dev - resources: - - mcads/status - verbs: - - get diff --git a/config/rbac/mcad_viewer_role.yaml b/config/rbac/mcad_viewer_role.yaml deleted file mode 100644 index df4526d2..00000000 --- a/config/rbac/mcad_viewer_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# permissions for end users to view mcads. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: mcad-viewer-role -rules: -- apiGroups: - - codeflare.codeflare.dev - resources: - - mcads - verbs: - - get - - list - - watch -- apiGroups: - - codeflare.codeflare.dev - resources: - - mcads/status - verbs: - - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index a2cc83f9..f8a12f4e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -56,58 +56,6 @@ rules: - subjectaccessreviews verbs: - create -- apiGroups: - - codeflare.codeflare.dev - resources: - - instascales - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - codeflare.codeflare.dev - resources: - - instascales/finalizers - verbs: - - update -- apiGroups: - - codeflare.codeflare.dev - resources: - - instascales/status - verbs: - - get - - patch - - update -- apiGroups: - - codeflare.codeflare.dev - resources: - - mcads - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - codeflare.codeflare.dev - resources: - - mcads/finalizers - verbs: - - update -- apiGroups: - - codeflare.codeflare.dev - resources: - - mcads/status - verbs: - - get - - patch - - update - apiGroups: - config.openshift.io resources: diff --git a/config/samples/codeflare_v1alpha1_instascale.yaml b/config/samples/codeflare_v1alpha1_instascale.yaml deleted file mode 100644 index 033cab0c..00000000 --- a/config/samples/codeflare_v1alpha1_instascale.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: codeflare.codeflare.dev/v1alpha1 -kind: InstaScale -metadata: - labels: - app.kubernetes.io/name: instascale - app.kubernetes.io/instance: instascale-sample - app.kubernetes.io/part-of: codeflare-operator - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/created-by: codeflare-operator - name: instascale-sample -spec: {} - # TODO(user): Add fields here diff --git a/config/samples/codeflare_v1alpha1_mcad.yaml b/config/samples/codeflare_v1alpha1_mcad.yaml deleted file mode 100644 index 624db497..00000000 --- a/config/samples/codeflare_v1alpha1_mcad.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: codeflare.codeflare.dev/v1alpha1 -kind: MCAD -metadata: - name: mcad-sample -spec: {} - # TODO(user): Add fields here diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml deleted file mode 100644 index 5a242aff..00000000 --- a/config/samples/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -## Append samples you want in your CSV to this file as resources ## -resources: -- codeflare_v1alpha1_mcad.yaml -- codeflare_v1alpha1_instascale.yaml -#+kubebuilder:scaffold:manifestskustomizesamples From 5bbaf617b42d4b4e946912a865699659b81c5f89 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 8 Sep 2023 11:43:02 +0200 Subject: [PATCH 128/377] Remove code generation verify workflow --- .github/workflows/verify_generated_files.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml index 304d9df0..aa85fcff 100644 --- a/.github/workflows/verify_generated_files.yml +++ b/.github/workflows/verify_generated_files.yml @@ -15,17 +15,6 @@ on: - '**go.mod' - '**go.sum' jobs: - verify-generated-functions: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set Go - uses: actions/setup-go@v3 - with: - go-version: v1.19 - - name: Verify that the DeepCopy, DeepCopyInto, and DeepCopyObject method implementations have been generated - run: make generate && git diff --exit-code - verify-imports: runs-on: ubuntu-latest steps: From c6afaf2fe2354cc2aca411dcea3b28a9974b6994 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 8 Sep 2023 14:32:15 +0200 Subject: [PATCH 129/377] Clean up operator roles --- config/rbac/role.yaml | 103 ++++++------------------------------------ 1 file changed, 14 insertions(+), 89 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f8a12f4e..1393982c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -18,18 +18,6 @@ rules: - patch - update - watch -- apiGroups: - - apps - resources: - - deployments - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - apps resources: @@ -97,23 +85,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - configmaps - - persistentvolumeclaims - - persistentvolumes - - secrets - - serviceaccounts - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - "" resources: @@ -141,8 +112,6 @@ rules: - apiGroups: - "" resources: - - lists - - namespaces - pods verbs: - create @@ -153,13 +122,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - - persistentvolumes - verbs: - - '*' - apiGroups: - "" resources: @@ -175,12 +137,6 @@ rules: - get - list - watch -- apiGroups: - - custom.metrics.k8s.io - resources: - - '*' - verbs: - - '*' - apiGroups: - events.k8s.io resources: @@ -190,14 +146,6 @@ rules: - create - patch - update -- apiGroups: - - extensions - resources: - - replicasets - verbs: - - get - - list - - watch - apiGroups: - machine.openshift.io resources: @@ -211,23 +159,9 @@ rules: - update - watch - apiGroups: - - policy - resources: - - poddisruptionbudgets - verbs: - - get - - list - - watch -- apiGroups: - - quota.codeflare.dev - - workload.codeflare.dev + - scheduling.sigs.k8s.io resources: - - appwrappers - - appwrappers/finalizers - - appwrappers/status - - queuejobs - - quotasubtrees - - schedulingspecs + - podgroups verbs: - create - delete @@ -238,34 +172,35 @@ rules: - update - watch - apiGroups: - - rbac.authorization.k8s.io + - storage.k8s.io resources: - - clusterrolebindings - - clusterroles + - csidrivers + - csinodes + - csistoragecapacities verbs: - - create - - delete - get - list - - update - watch - apiGroups: - - rbac.authorization.k8s.io + - workload.codeflare.dev resources: - - rolebindings - - roles + - appwrappers + - appwrappers/finalizers + - appwrappers/status + - schedulingspecs verbs: - create - delete + - deletecollection - get - list - patch - update - watch - apiGroups: - - scheduling.sigs.k8s.io + - quota.codeflare.dev resources: - - podgroups + - quotasubtrees verbs: - create - delete @@ -275,13 +210,3 @@ rules: - patch - update - watch -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - - csinodes - - csistoragecapacities - verbs: - - get - - list - - watch From 083ad2b6d39371fecf70fc45f26e9fc3901e5774 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 13 Sep 2023 15:20:54 +0200 Subject: [PATCH 130/377] Use structured configuration --- .github/workflows/e2e_tests.yaml | 3 -- .github/workflows/olm_tests.yaml | 3 -- Dockerfile | 3 +- config/default/instascale-configmap.yaml | 6 --- config/default/kustomization.yaml | 2 - config/manager/manager.yaml | 2 - go.mod | 13 ++--- go.sum | 8 +-- main.go | 69 +++++++++++------------- pkg/config/config.go | 46 ++++++++++++++++ 10 files changed, 87 insertions(+), 68 deletions(-) delete mode 100644 config/default/instascale-configmap.yaml create mode 100644 pkg/config/config.go diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 6abb3e84..42ee0b73 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -73,9 +73,6 @@ jobs: echo Deploying CodeFlare operator IMG="${REGISTRY_ADDRESS}"/codeflare-operator make image-push -e IMG="${IMG}" - # FIXME - kubectl create ns openshift-operators - kubectl apply -n openshift-operators -f config/default/instascale-configmap.yaml make deploy -e IMG="${IMG}" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 405c74a9..6ecede4e 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -117,9 +117,6 @@ jobs: - name: Deploy CodeFlare stack run: | - # FIXME - kubectl apply -n openshift-operators -f config/default/instascale-configmap.yaml - make setup-e2e - name: Build operator and catalog image diff --git a/Dockerfile b/Dockerfile index b896f03b..b949b1bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,8 +7,9 @@ COPY go.mod go.mod COPY go.sum go.sum RUN go mod download -# Copy the go source +# Copy the Go sources COPY main.go main.go +COPY pkg/ pkg/ # Build USER root diff --git a/config/default/instascale-configmap.yaml b/config/default/instascale-configmap.yaml deleted file mode 100644 index 9ff6c095..00000000 --- a/config/default/instascale-configmap.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: instascale-config -data: - maxScaleoutAllowed: "15" diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index d3937269..ad203a94 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -28,8 +28,6 @@ bases: resources: # Add metrics service - metrics_service.yaml -# TODO: refactor configuration -- instascale-configmap.yaml # Mount the controller config file for loading manager configurations # through a ComponentConfig type diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 519ecc1c..bd32cd35 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -37,8 +37,6 @@ spec: args: - "--health-probe-bind-address=:8081" - "--metrics-bind-address=0.0.0.0:8082" - - "--leader-elect" - - "--configs-namespace=openshift-operators" image: controller:latest imagePullPolicy: Always name: manager diff --git a/go.mod b/go.mod index 3cacfb8f..dc909581 100644 --- a/go.mod +++ b/go.mod @@ -7,20 +7,19 @@ require ( github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/instascale v0.0.0-00010101000000-000000000000 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230904071404-ce734452edc6 + github.com/project-codeflare/instascale v0.0.9-0.20230913044449-3db3f4b4e9b1 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230913035103-6e7bbdc8fb1f github.com/ray-project/kuberay/ray-operator v0.0.0-20230908233208-a8f730e5a2b6 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 k8s.io/client-go v0.26.3 + k8s.io/component-base v0.26.2 + k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 sigs.k8s.io/controller-runtime v0.14.6 ) -replace ( - github.com/project-codeflare/instascale => github.com/Bobbins228/instascale v0.0.0-20230724130835-2a8ebf58690f - sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 -) +replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 require ( github.com/NYTimes/gziphandler v1.1.1 // indirect @@ -125,13 +124,11 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.26.1 // indirect k8s.io/apiserver v0.26.2 // indirect - k8s.io/component-base v0.26.2 // indirect k8s.io/klog v1.0.0 // indirect k8s.io/klog/v2 v2.90.1 // indirect k8s.io/kms v0.26.2 // indirect k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d // indirect k8s.io/metrics v0.26.2 // indirect - k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 // indirect sigs.k8s.io/custom-metrics-apiserver v0.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index 60247273..89bbaf8f 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Bobbins228/instascale v0.0.0-20230724130835-2a8ebf58690f h1:aEtM6OZ2dCY97VnVsVYZl2GJOe+6WegPGLgLXQt5BGk= -github.com/Bobbins228/instascale v0.0.0-20230724130835-2a8ebf58690f/go.mod h1:kwBkC+o4W+a1dpftXbVQjvGCVyu2206HlZtcp1+ZjyY= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -460,8 +458,10 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230904071404-ce734452edc6 h1:YT6C0Z84TgftYcoNUd/dwvXxGJvXy67EeWAMQjpKvdU= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230904071404-ce734452edc6/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/instascale v0.0.9-0.20230913044449-3db3f4b4e9b1 h1:SNc05sEO/KBuRwep+u7J/ZrT06IhncjTHBvopc4LDlU= +github.com/project-codeflare/instascale v0.0.9-0.20230913044449-3db3f4b4e9b1/go.mod h1:n2fIzs06LmGZFB3vRfQcXOkPebN3Hwd5Y+SD5v0hnrs= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230913035103-6e7bbdc8fb1f h1:xrhSm18A/Gvmm9cia05eAGkj0i9fTMqeTExz8rQTHJU= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230913035103-6e7bbdc8fb1f/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= diff --git a/main.go b/main.go index 150547f5..2c2c0435 100644 --- a/main.go +++ b/main.go @@ -22,8 +22,8 @@ import ( "time" instascale "github.com/project-codeflare/instascale/controllers" - mcadoptions "github.com/project-codeflare/multi-cluster-app-dispatcher/cmd/kar-controllers/app/options" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + mcadconfig "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config" mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob" "go.uber.org/zap/zapcore" @@ -31,9 +31,13 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" + configv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + "github.com/project-codeflare/codeflare-operator/pkg/config" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. // +kubebuilder:scaffold:imports @@ -51,73 +55,60 @@ func init() { func main() { var metricsAddr string - var enableLeaderElection bool var probeAddr string - var configsNamespace string - var ocmSecretNamespace string - // Operator flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") - - // InstScale - flag.StringVar(&configsNamespace, "configs-namespace", "kube-system", "The namespace containing the Instacale configmap") - flag.StringVar(&ocmSecretNamespace, "ocm-secret-namespace", "default", "The namespace containing the OCM secret") - - mcadOptions := mcadoptions.NewServerOption() zapOptions := zap.Options{ Development: true, TimeEncoder: zapcore.TimeEncoderOfLayout(time.RFC3339), } - - flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError) - flag.CommandLine.VisitAll(func(f *flag.Flag) { - if f.Name != "kubeconfig" { - flagSet.Var(f.Value, f.Name, f.Usage) - } - }) - - zapOptions.BindFlags(flagSet) - mcadOptions.AddFlags(flagSet) - _ = flagSet.Parse(os.Args[1:]) + zapOptions.BindFlags(flag.CommandLine) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) ctx := ctrl.SetupSignalHandler() + cfg := config.CodeFlareOperatorConfiguration{ + LeaderElection: &configv1alpha1.LeaderElectionConfiguration{}, + MCAD: &mcadconfig.MCADConfiguration{}, + InstaScale: &config.InstaScaleConfiguration{}, + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - Port: 9443, - HealthProbeBindAddress: probeAddr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "5a3ca514.codeflare.dev", + Scheme: scheme, + MetricsBindAddress: metricsAddr, + HealthProbeBindAddress: probeAddr, + LeaderElection: pointer.BoolDeref(cfg.LeaderElection.LeaderElect, false), + LeaderElectionID: cfg.LeaderElection.ResourceName, + LeaderElectionNamespace: cfg.LeaderElection.ResourceNamespace, + LeaderElectionResourceLock: cfg.LeaderElection.ResourceLock, + LeaseDuration: &cfg.LeaderElection.LeaseDuration.Duration, + RetryPeriod: &cfg.LeaderElection.RetryPeriod.Duration, + RenewDeadline: &cfg.LeaderElection.RenewDeadline.Duration, }) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } - mcadQueueController := mcad.NewJobController(mgr.GetConfig(), mcadOptions) + mcadQueueController := mcad.NewJobController(mgr.GetConfig(), cfg.MCAD, &mcadconfig.MCADConfigurationExtended{}) if mcadQueueController == nil { // FIXME: update NewJobController so it follows Go idiomatic error handling and return an error instead of a nil object os.Exit(1) } mcadQueueController.Run(ctx.Done()) - instascaleController := &instascale.AppWrapperReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - ConfigsNamespace: configsNamespace, - OcmSecretNamespace: ocmSecretNamespace, + if pointer.BoolDeref(cfg.InstaScale.Enabled, false) { + instaScaleController := &instascale.AppWrapperReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Config: cfg.InstaScale.InstaScaleConfiguration, + } + exitOnError(instaScaleController.SetupWithManager(mgr), "Error setting up InstaScale controller") } - exitOnError(instascaleController.SetupWithManager(mgr), "Error setting up InstaScale controller") - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 00000000..55bf2554 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,46 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + instascale "github.com/project-codeflare/instascale/pkg/config" + mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config" + + configv1alpha1 "k8s.io/component-base/config/v1alpha1" +) + +type CodeFlareOperatorConfiguration struct { + // LeaderElection is the LeaderElection config to be used when configuring + // the manager.Manager leader election + LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"` + + // The MCAD controller configuration + MCAD *mcad.MCADConfiguration `json:"mcad,omitempty"` + + // The InstaScale controller configuration + InstaScale *InstaScaleConfiguration `json:"instascale,omitempty"` +} + +type InstaScaleConfiguration struct { + // enabled controls whether the InstaScale controller is started. + // It may default to true on platforms that InstaScale supports. + // Otherwise, it defaults to false. + Enabled *bool `json:"enabled,omitempty"` + + // The InstaScale controller configuration + instascale.InstaScaleConfiguration `json:",inline,omitempty"` +} From 299166146eb4f471632c9cbf764f1b4480bd1988 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 13 Sep 2023 15:23:16 +0200 Subject: [PATCH 131/377] Add quota API types to runtime scheme --- main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 2c2c0435..e807c4b4 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,7 @@ import ( instascale "github.com/project-codeflare/instascale/controllers" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + quotasubtreev1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/quotaplugins/quotasubtree/v1" mcadconfig "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config" mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob" "go.uber.org/zap/zapcore" @@ -50,7 +51,8 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - _ = mcadv1beta1.AddToScheme(scheme) + utilruntime.Must(mcadv1beta1.AddToScheme(scheme)) + utilruntime.Must(quotasubtreev1.AddToScheme(scheme)) } func main() { From 7857bb86b964e93f09df50ad452620a61c791505 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 18 Sep 2023 09:38:46 +0200 Subject: [PATCH 132/377] Add controller manager structured config --- config/manager/manager.yaml | 3 -- main.go | 51 +++++++++++++++----------------- pkg/config/config.go | 58 +++++++++++++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 34 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index bd32cd35..87020106 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -34,9 +34,6 @@ spec: containers: - command: - /manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=0.0.0.0:8082" image: controller:latest imagePullPolicy: Always name: manager diff --git a/main.go b/main.go index e807c4b4..347271ad 100644 --- a/main.go +++ b/main.go @@ -32,6 +32,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/rest" configv1alpha1 "k8s.io/component-base/config/v1alpha1" "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" @@ -56,12 +57,6 @@ func init() { } func main() { - var metricsAddr string - var probeAddr string - - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - zapOptions := zap.Options{ Development: true, TimeEncoder: zapcore.TimeEncoderOfLayout(time.RFC3339), @@ -73,15 +68,27 @@ func main() { ctx := ctrl.SetupSignalHandler() cfg := config.CodeFlareOperatorConfiguration{ - LeaderElection: &configv1alpha1.LeaderElectionConfiguration{}, - MCAD: &mcadconfig.MCADConfiguration{}, - InstaScale: &config.InstaScaleConfiguration{}, + ControllerManager: config.ControllerManager{ + LeaderElection: &configv1alpha1.LeaderElectionConfiguration{}, + }, + MCAD: &mcadconfig.MCADConfiguration{}, + InstaScale: &config.InstaScaleConfiguration{}, } - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + kubeConfig, err := ctrl.GetConfig() + exitOnError(err, "unable to get client config") + + if kubeConfig.UserAgent == "" { + kubeConfig.UserAgent = "codeflare-operator" + } + kubeConfig.Burst = int(pointer.Int32Deref(cfg.ClientConnection.Burst, int32(rest.DefaultBurst))) + kubeConfig.QPS = pointer.Float32Deref(cfg.ClientConnection.QPS, rest.DefaultQPS) + setupLog.V(2).Info("REST client", "qps", kubeConfig.QPS, "burst", kubeConfig.Burst) + + mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{ Scheme: scheme, - MetricsBindAddress: metricsAddr, - HealthProbeBindAddress: probeAddr, + MetricsBindAddress: cfg.Metrics.BindAddress, + HealthProbeBindAddress: cfg.Health.BindAddress, LeaderElection: pointer.BoolDeref(cfg.LeaderElection.LeaderElect, false), LeaderElectionID: cfg.LeaderElection.ResourceName, LeaderElectionNamespace: cfg.LeaderElection.ResourceNamespace, @@ -90,10 +97,7 @@ func main() { RetryPeriod: &cfg.LeaderElection.RetryPeriod.Duration, RenewDeadline: &cfg.LeaderElection.RenewDeadline.Duration, }) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } + exitOnError(err, "unable to start manager") mcadQueueController := mcad.NewJobController(mgr.GetConfig(), cfg.MCAD, &mcadconfig.MCADConfigurationExtended{}) if mcadQueueController == nil { @@ -111,20 +115,11 @@ func main() { exitOnError(instaScaleController.SetupWithManager(mgr), "Error setting up InstaScale controller") } - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } + exitOnError(mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping), "unable to set up health check") + exitOnError(mgr.AddReadyzCheck(cfg.Health.ReadinessEndpointName, healthz.Ping), "unable to set up ready check") setupLog.Info("starting manager") - if err := mgr.Start(ctx); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } + exitOnError(mgr.Start(ctx), "error running manager") } func exitOnError(err error, msg string) { diff --git a/pkg/config/config.go b/pkg/config/config.go index 55bf2554..295e3e00 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -24,9 +24,12 @@ import ( ) type CodeFlareOperatorConfiguration struct { - // LeaderElection is the LeaderElection config to be used when configuring - // the manager.Manager leader election - LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"` + // ClientConnection provides additional configuration options for Kubernetes + // API server client. + ClientConnection *ClientConnection `json:"clientConnection,omitempty"` + + // ControllerManager returns the configurations for controllers + ControllerManager `json:",inline"` // The MCAD controller configuration MCAD *mcad.MCADConfiguration `json:"mcad,omitempty"` @@ -44,3 +47,52 @@ type InstaScaleConfiguration struct { // The InstaScale controller configuration instascale.InstaScaleConfiguration `json:",inline,omitempty"` } + +type ControllerManager struct { + // Metrics contains the controller metrics configuration + // +optional + Metrics MetricsConfiguration `json:"metrics,omitempty"` + + // Health contains the controller health configuration + // +optional + Health HealthConfiguration `json:"health,omitempty"` + + // LeaderElection is the LeaderElection config to be used when configuring + // the manager.Manager leader election + LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"` +} + +type ClientConnection struct { + // QPS controls the number of queries per second allowed before client-side throttling + // connection to the API server. + QPS *float32 `json:"qps,omitempty"` + + // Burst allows extra queries to accumulate when a client is exceeding its rate. + Burst *int32 `json:"burst,omitempty"` +} + +// MetricsConfiguration defines the metrics configuration. +type MetricsConfiguration struct { + // BindAddress is the TCP address that the controller should bind to + // for serving Prometheus metrics. + // It can be set to "0" to disable the metrics serving. + // +optional + BindAddress string `json:"bindAddress,omitempty"` +} + +// HealthConfiguration defines the health configuration. +type HealthConfiguration struct { + // BindAddress is the TCP address that the controller should bind to + // for serving health probes. + // It can be set to "0" or "" to disable serving the health probe. + // +optional + BindAddress string `json:"bindAddress,omitempty"` + + // ReadinessEndpointName, defaults to "readyz" + // +optional + ReadinessEndpointName string `json:"readinessEndpointName,omitempty"` + + // LivenessEndpointName, defaults to "healthz" + // +optional + LivenessEndpointName string `json:"livenessEndpointName,omitempty"` +} From 73dcb21901962c8713a0c1454cfc1b86f88dc8d6 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 18 Sep 2023 12:16:54 +0200 Subject: [PATCH 133/377] Init and load operator ConfigMap --- config/default/kustomization.yaml | 49 --------- config/default/manager_config_patch.yaml | 20 ---- config/manager/controller_manager_config.yaml | 21 ---- config/manager/kustomization.yaml | 7 -- config/manager/manager.yaml | 5 + go.mod | 2 +- main.go | 103 +++++++++++++++++- 7 files changed, 105 insertions(+), 102 deletions(-) delete mode 100644 config/default/manager_config_patch.yaml delete mode 100644 config/manager/controller_manager_config.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index ad203a94..870c75a5 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -17,57 +17,8 @@ bases: - ../crd - ../rbac - ../manager -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -# - ../webhook -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -# - ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. # - ../prometheus resources: -# Add metrics service - metrics_service.yaml - -# Mount the controller config file for loading manager configurations -# through a ComponentConfig type -# - manager_config_patch.yaml - -# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in -# crd/kustomization.yaml -# - manager_webhook_patch.yaml - -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. -# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. -# 'CERTMANAGER' needs to be enabled to use ca injection -# - webhookcainjection_patch.yaml - -# the following config is for teaching kustomize how to do var substitution -vars: -# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. -# - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldref: -# fieldpath: metadata.namespace -# - name: CERTIFICATE_NAME -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# - name: SERVICE_NAMESPACE # namespace of the service -# objref: -# kind: Service -# version: v1 -# name: webhook-service -# fieldref: -# fieldpath: metadata.namespace -# - name: SERVICE_NAME -# objref: -# kind: Service -# version: v1 -# name: webhook-service diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml deleted file mode 100644 index 98456c9a..00000000 --- a/config/default/manager_config_patch.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: manager - namespace: system -spec: - template: - spec: - containers: - - name: manager - args: - - "--config=controller_manager_config.yaml" - volumeMounts: - - name: manager-config - mountPath: /controller_manager_config.yaml - subPath: controller_manager_config.yaml - volumes: - - name: manager-config - configMap: - name: manager-config diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml deleted file mode 100644 index 21133591..00000000 --- a/config/manager/controller_manager_config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 -kind: ControllerManagerConfig -health: - healthProbeBindAddress: :8081 -metrics: - bindAddress: 0.0.0.0:8082 -webhook: - port: 9443 -leaderElection: - leaderElect: true - resourceName: 5a3ca514.codeflare.dev -# leaderElectionReleaseOnCancel defines if the leader should step down volume -# when the Manager ends. This requires the binary to immediately end when the -# Manager is stopped, otherwise, this setting is unsafe. Setting this significantly -# speeds up voluntary leader transitions as the new leader don't have to wait -# LeaseDuration time first. -# In the default scaffold provided, the program ends immediately after -# the manager stops, so would be fine to enable this option. However, -# if you are doing or is intended to do any operation such as perform cleanups -# after the manager stops then its usage might be unsafe. -# leaderElectionReleaseOnCancel: true diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 538c69f2..4fe5ccaa 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -3,10 +3,3 @@ resources: generatorOptions: disableNameSuffixHash: true - -configMapGenerator: -- files: - - controller_manager_config.yaml - name: manager-config -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 87020106..ca1823c3 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -42,6 +42,11 @@ spec: capabilities: drop: - "ALL" + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace ports: - containerPort: 8080 protocol: TCP diff --git a/go.mod b/go.mod index dc909581..5dbd8dde 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( k8s.io/component-base v0.26.2 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 sigs.k8s.io/controller-runtime v0.14.6 + sigs.k8s.io/yaml v1.3.0 ) replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 @@ -133,5 +134,4 @@ require ( sigs.k8s.io/custom-metrics-apiserver v0.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect - sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/main.go b/main.go index 347271ad..9c157ebc 100644 --- a/main.go +++ b/main.go @@ -17,19 +17,27 @@ limitations under the License. package main import ( + "context" "flag" + "fmt" "os" + "strings" "time" instascale "github.com/project-codeflare/instascale/controllers" + instascaleconfig "github.com/project-codeflare/instascale/pkg/config" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" quotasubtreev1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/quotaplugins/quotasubtree/v1" mcadconfig "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config" mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob" "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/rest" @@ -38,6 +46,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/yaml" "github.com/project-codeflare/codeflare-operator/pkg/config" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -57,6 +66,11 @@ func init() { } func main() { + var configMapName string + flag.StringVar(&configMapName, "config", "codeflare-operator-config", + "The name of the ConfigMap to load the operator configuration from. "+ + "If it does not exist, the operator will create and initialise it.") + zapOptions := zap.Options{ Development: true, TimeEncoder: zapcore.TimeEncoderOfLayout(time.RFC3339), @@ -67,20 +81,41 @@ func main() { ctx := ctrl.SetupSignalHandler() - cfg := config.CodeFlareOperatorConfiguration{ + cfg := &config.CodeFlareOperatorConfiguration{ + ClientConnection: &config.ClientConnection{ + QPS: pointer.Float32(50), + Burst: pointer.Int32(100), + }, ControllerManager: config.ControllerManager{ + Metrics: config.MetricsConfiguration{ + BindAddress: ":8080", + }, + Health: config.HealthConfiguration{ + BindAddress: ":8081", + ReadinessEndpointName: "readyz", + LivenessEndpointName: "healthz", + }, LeaderElection: &configv1alpha1.LeaderElectionConfiguration{}, }, - MCAD: &mcadconfig.MCADConfiguration{}, - InstaScale: &config.InstaScaleConfiguration{}, + MCAD: &mcadconfig.MCADConfiguration{}, + InstaScale: &config.InstaScaleConfiguration{ + Enabled: pointer.Bool(false), + InstaScaleConfiguration: instascaleconfig.InstaScaleConfiguration{ + MaxScaleoutAllowed: 5, + }, + }, } kubeConfig, err := ctrl.GetConfig() exitOnError(err, "unable to get client config") - if kubeConfig.UserAgent == "" { kubeConfig.UserAgent = "codeflare-operator" } + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + exitOnError(err, "unable to create Kubernetes client") + + exitOnError(loadIntoOrCreate(ctx, kubeClient, namespaceOrDie(), configMapName, cfg), "unable to initialise configuration") + kubeConfig.Burst = int(pointer.Int32Deref(cfg.ClientConnection.Burst, int32(rest.DefaultBurst))) kubeConfig.QPS = pointer.Float32Deref(cfg.ClientConnection.QPS, rest.DefaultQPS) setupLog.V(2).Info("REST client", "qps", kubeConfig.QPS, "burst", kubeConfig.Burst) @@ -122,6 +157,66 @@ func main() { exitOnError(mgr.Start(ctx), "error running manager") } +func loadIntoOrCreate(ctx context.Context, client kubernetes.Interface, ns, name string, cfg *config.CodeFlareOperatorConfiguration) error { + configMap, err := client.CoreV1().ConfigMaps(ns).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return createConfigMap(ctx, client, ns, name, cfg) + } else if err != nil { + return err + } + + if len(configMap.Data) != 1 { + return fmt.Errorf("cannot resolve config from ConfigMap %s/%s", configMap.Namespace, configMap.Name) + } + + for _, data := range configMap.Data { + return yaml.Unmarshal([]byte(data), cfg) + } + + return nil +} + +func createConfigMap(ctx context.Context, client kubernetes.Interface, ns, name string, cfg *config.CodeFlareOperatorConfiguration) error { + content, err := yaml.Marshal(cfg) + if err != nil { + return err + } + + configMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Data: map[string]string{ + "config.yaml": string(content), + }, + } + + _, err = client.CoreV1().ConfigMaps(ns).Create(ctx, configMap, metav1.CreateOptions{}) + return err +} + +func namespaceOrDie() string { + // This way assumes you've set the NAMESPACE environment variable either manually, when running + // the operator standalone, or using the downward API, when running the operator in-cluster. + if ns := os.Getenv("NAMESPACE"); ns != "" { + return ns + } + + // Fall back to the namespace associated with the service account token, if available + if data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil { + if ns := strings.TrimSpace(string(data)); len(ns) > 0 { + return ns + } + } + + panic("unable to determine current namespace") +} + func exitOnError(err error, msg string) { if err != nil { setupLog.Error(err, msg) From 2ae625d0db8077fe89fa8aee3fcc01ef84bafb0d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 19 Sep 2023 19:12:05 +0200 Subject: [PATCH 134/377] Remove openshift-goimports from tools.go --- hack/tools.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/hack/tools.go b/hack/tools.go index f23ba362..f28d1037 100644 --- a/hack/tools.go +++ b/hack/tools.go @@ -2,8 +2,3 @@ // +build tools package hack - -// Add tools that hack scripts depend on here, to ensure they are vendored. -import ( - goimports "github.com/openshift-eng/openshift-goimports" -) From bba2c571a3e5a2ece530ea4498901d6446489d7b Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 19 Sep 2023 18:54:27 +0200 Subject: [PATCH 135/377] Update Go modules target in Makefile --- Makefile | 20 +++------ go.mod | 19 ++------- go.sum | 127 +++---------------------------------------------------- 3 files changed, 16 insertions(+), 150 deletions(-) diff --git a/Makefile b/Makefile index 3c0c2e13..78d31374 100644 --- a/Makefile +++ b/Makefile @@ -12,15 +12,13 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.0.8 +INSTASCALE_VERSION ?= main +INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.34.1 -# MCAD_REF, MCAD_REPO and MCAD_CRD define the reference to MCAD CRD resources -MCAD_REF ?= release-${MCAD_VERSION} +MCAD_VERSION ?= main MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) -# The image is still published using the MCAD_REF format (i.e release-vX.Y.Z) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) @@ -66,12 +64,6 @@ IMAGE_ORG_BASE ?= quay.io/project-codeflare # codeflare.dev/codeflare-operator-bundle:$VERSION and codeflare.dev/codeflare-operator-catalog:$VERSION. IMAGE_TAG_BASE ?= $(IMAGE_ORG_BASE)/codeflare-operator -# MCAD_IMAGE defines the default container image for the MCAD controller -MCAD_IMAGE ?= $(IMAGE_ORG_BASE)/mcad-controller:$(MCAD_REF) - -# INSTASCALE_IMAGE defines the default container image for the InstaScale controller -INSTASCALE_IMAGE ?= $(IMAGE_ORG_BASE)/instascale-controller:$(INSTASCALE_VERSION) - # RAY_IMAGE defines the default container image for Ray (used for testing) RAY_IMAGE ?= rayproject/ray:$(RAY_VERSION) @@ -167,8 +159,10 @@ vet: ## Run go vet against code. .PHONY: modules modules: ## Update Go dependencies. - #go get $(MCAD_REPO)@$(MCAD_VERSION) - #go get github.com/ray-project/kuberay/ray-operator + go get $(MCAD_REPO)@$(MCAD_VERSION) + go get $(INSTASCALE_REPO)@$(INSTASCALE_VERSION) + go get github.com/ray-project/kuberay/ray-operator + go mod tidy .PHONY: build build: modules defaults fmt vet ## Build manager binary. diff --git a/go.mod b/go.mod index 5dbd8dde..6523b87c 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,11 @@ go 1.19 require ( github.com/onsi/gomega v1.27.10 - github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/instascale v0.0.9-0.20230913044449-3db3f4b4e9b1 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230913035103-6e7bbdc8fb1f - github.com/ray-project/kuberay/ray-operator v0.0.0-20230908233208-a8f730e5a2b6 + github.com/project-codeflare/instascale v0.0.9-0.20230914113244-30c8b74bdb6b + github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230918125534-b445015360ab + github.com/ray-project/kuberay/ray-operator v0.0.0-20230916205808-6d5020fba579 go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 @@ -60,35 +59,25 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/magiconair/properties v1.8.4 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/microcosm-cc/bluemonday v1.0.18 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift-online/ocm-sdk-go v0.1.327 // indirect - github.com/pelletier/go-toml v1.8.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - github.com/spf13/afero v1.4.1 // indirect - github.com/spf13/cast v1.3.1 // indirect github.com/spf13/cobra v1.7.0 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.7.1 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect - github.com/subosito/gotenv v1.2.0 // indirect go.etcd.io/etcd/api/v3 v3.5.5 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect go.etcd.io/etcd/client/v3 v3.5.5 // indirect @@ -105,7 +94,6 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/crypto v0.11.0 // indirect - golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect golang.org/x/sync v0.2.0 // indirect @@ -119,7 +107,6 @@ require ( google.golang.org/grpc v1.49.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 89bbaf8f..19f02131 100644 --- a/go.sum +++ b/go.sum @@ -22,7 +22,6 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -48,9 +47,6 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -59,8 +55,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -84,23 +78,18 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -122,7 +111,6 @@ github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= @@ -164,7 +152,6 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= @@ -172,7 +159,6 @@ github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -242,45 +228,22 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -341,7 +304,6 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -353,17 +315,13 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -379,15 +337,10 @@ github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQSeYtvkY= -github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= @@ -398,18 +351,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09 github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -424,7 +365,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= @@ -438,32 +378,23 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= -github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2 h1:Zq1BYSO2UmZuu/O1tpYIaC/7ir7ljFqdEY90TwqlseI= -github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2/go.mod h1:VV07Ee+14Mjpwh/ZxtwgAieiCvZ20YVUB9jLHixzswk= github.com/openshift-online/ocm-sdk-go v0.1.327 h1:WR822bGdQoMuZ2+dFdhZz3fpD2NlJhGr+F3FJPXvqFU= github.com/openshift-online/ocm-sdk-go v0.1.327/go.mod h1:KYOw8kAKAHyPrJcQoVR82CneQ4ofC02Na4cXXaTq4Nw= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/project-codeflare/instascale v0.0.9-0.20230913044449-3db3f4b4e9b1 h1:SNc05sEO/KBuRwep+u7J/ZrT06IhncjTHBvopc4LDlU= -github.com/project-codeflare/instascale v0.0.9-0.20230913044449-3db3f4b4e9b1/go.mod h1:n2fIzs06LmGZFB3vRfQcXOkPebN3Hwd5Y+SD5v0hnrs= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230913035103-6e7bbdc8fb1f h1:xrhSm18A/Gvmm9cia05eAGkj0i9fTMqeTExz8rQTHJU= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230913035103-6e7bbdc8fb1f/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/instascale v0.0.9-0.20230914113244-30c8b74bdb6b h1:xLTlkfejA1xOIxHKFAqGIbXmN9eOLC5jLAowhdydKI4= +github.com/project-codeflare/instascale v0.0.9-0.20230914113244-30c8b74bdb6b/go.mod h1:n2fIzs06LmGZFB3vRfQcXOkPebN3Hwd5Y+SD5v0hnrs= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230918125534-b445015360ab h1:7OM2Kp1+2HtE7p6r5YrfHOCCgAxzEebFP9VLsPTzwQ8= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230918125534-b445015360ab/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= @@ -477,8 +408,6 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= @@ -486,26 +415,21 @@ github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+ github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230908233208-a8f730e5a2b6 h1:NP7S29MtvEH4zD0oawHqGE/DXu5uYbGvXQq6mh5+3Kk= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230908233208-a8f730e5a2b6/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230916205808-6d5020fba579 h1:cNHsu7dO/vK/EFxEutzQXk4+aJCFnI9qef8Krbhw2Fs= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230916205808-6d5020fba579/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -513,29 +437,12 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ= -github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= -github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -552,15 +459,11 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -568,7 +471,6 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= @@ -628,7 +530,6 @@ go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -676,15 +577,10 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -749,12 +645,9 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -833,14 +726,12 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -855,7 +746,6 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1003,15 +893,10 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 87d9561415c24cc4e7e64bb001e5ddab9b120c54 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Wed, 20 Sep 2023 15:08:35 +0100 Subject: [PATCH 136/377] ci: update release workflow to remove sed edits to instascale testcase deployments --- .github/workflows/tag-and-build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 1a322d70..0e87a981 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -98,8 +98,6 @@ jobs: sed -i -E "s/(.*MCAD_REF \?= ).*/\1release-\${MCAD_VERSION}/" Makefile sed -i -E "s/(.*CODEFLARE_SDK_VERSION \?= ).*/\1$CODEFLARE_SDK_VERSION/" Makefile sed -i -E "s/(.*INSTASCALE_VERSION \?= ).*/\1${{ github.event.inputs.instascale-version }}/" Makefile - sed -i -E "s/(.*instascale-controller:).*/\1${{ github.event.inputs.instascale-version }}/" controllers/testdata/instascale_test_results/case_1/deployment.yaml - sed -i -E "s/(.*instascale-controller:).*/\1${{ github.event.inputs.instascale-version }}/" controllers/testdata/instascale_test_results/case_2/deployment.yaml - name: Login to Quay.io uses: redhat-actions/podman-login@v1 From 2a8bcfe614b552cc911914bdaf25af008c560fc8 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account <138894154+codeflare-machine-account@users.noreply.github.com> Date: Wed, 20 Sep 2023 10:49:25 -0400 Subject: [PATCH 137/377] Update dependency versions for release v1.0.0-rc.1 (#296) Co-authored-by: dimakis --- Makefile | 6 +++--- README.md | 8 ++++---- go.mod | 6 +++--- go.sum | 12 ++++++------ test/support/defaults.go | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 78d31374..ba1e4c26 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= main +INSTASCALE_VERSION ?= v0.0.9 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= main +MCAD_VERSION ?= v1.35.0 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} @@ -28,7 +28,7 @@ KUBERAY_VERSION ?= v0.6.0 RAY_VERSION ?= 2.5.0 # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK -CODEFLARE_SDK_VERSION ?= 0.7.1 +CODEFLARE_SDK_VERSION ?= 0.8.0 # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to diff --git a/README.md b/README.md index 9924ee3b..a05315ae 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v0.2.3](https://github.com/project-codeflare/codeflare-operator/releases/tag/v0.2.3) | -| Multi-Cluster App Dispatcher | [v1.34.1](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.34.1) | -| CodeFlare-SDK | [v0.7.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.7.1) | -| InstaScale | [v0.0.8](https://github.com/project-codeflare/instascale/releases/tag/v0.0.8) | +| CodeFlare Operator | [v1.0.0-rc.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.1) | +| Multi-Cluster App Dispatcher | [v1.35.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.35.0) | +| CodeFlare-SDK | [v0.8.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.8.0) | +| InstaScale | [v0.0.9](https://github.com/project-codeflare/instascale/releases/tag/v0.0.9) | | KubeRay | [v0.5.0](https://github.com/ray-project/kuberay/releases/tag/v0.5.0) | diff --git a/go.mod b/go.mod index 6523b87c..78014907 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,9 @@ require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/instascale v0.0.9-0.20230914113244-30c8b74bdb6b - github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230918125534-b445015360ab - github.com/ray-project/kuberay/ray-operator v0.0.0-20230916205808-6d5020fba579 + github.com/project-codeflare/instascale v0.0.9 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.0 + github.com/ray-project/kuberay/ray-operator v0.0.0-20230920050528-dfdc51a88b6d go.uber.org/zap v1.24.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index 19f02131..a2f26185 100644 --- a/go.sum +++ b/go.sum @@ -390,10 +390,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/instascale v0.0.9-0.20230914113244-30c8b74bdb6b h1:xLTlkfejA1xOIxHKFAqGIbXmN9eOLC5jLAowhdydKI4= -github.com/project-codeflare/instascale v0.0.9-0.20230914113244-30c8b74bdb6b/go.mod h1:n2fIzs06LmGZFB3vRfQcXOkPebN3Hwd5Y+SD5v0hnrs= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230918125534-b445015360ab h1:7OM2Kp1+2HtE7p6r5YrfHOCCgAxzEebFP9VLsPTzwQ8= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230918125534-b445015360ab/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/instascale v0.0.9 h1:XlwprvuJNbNVnLxCa6lPz5bKUEMhJprVRumd42sRhVY= +github.com/project-codeflare/instascale v0.0.9/go.mod h1:fpc8TiFvR2r0duKFbKmGTshEGa0eGK9noWWfjhPxphE= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.0 h1:7Ks2+6Jd8tnwymSmSrwNb/CpAPmgS0lNIhMlj6DrT+o= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -421,8 +421,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230916205808-6d5020fba579 h1:cNHsu7dO/vK/EFxEutzQXk4+aJCFnI9qef8Krbhw2Fs= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230916205808-6d5020fba579/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230920050528-dfdc51a88b6d h1:qrCTwTnba5lQMiV6dYimR8epl8Ju90U+QLbmRQ4pe9k= +github.com/ray-project/kuberay/ray-operator v0.0.0-20230920050528-dfdc51a88b6d/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= diff --git a/test/support/defaults.go b/test/support/defaults.go index 66f36484..c93a3f48 100644 --- a/test/support/defaults.go +++ b/test/support/defaults.go @@ -5,7 +5,7 @@ package support // *********************** const ( - CodeFlareSDKVersion = "0.7.1" + CodeFlareSDKVersion = "0.8.0" RayVersion = "2.5.0" RayImage = "rayproject/ray:2.5.0" ) From b191a167a95b06e586b96cc0118d2b9bf41da41d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 14:50:36 +0000 Subject: [PATCH 138/377] build(deps): bump go.uber.org/zap from 1.24.0 to 1.26.0 Bumps [go.uber.org/zap](https://github.com/uber-go/zap) from 1.24.0 to 1.26.0. - [Release notes](https://github.com/uber-go/zap/releases) - [Changelog](https://github.com/uber-go/zap/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber-go/zap/compare/v1.24.0...v1.26.0) --- updated-dependencies: - dependency-name: go.uber.org/zap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 5 ++--- go.sum | 9 ++++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 78014907..c1caf36c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/project-codeflare/instascale v0.0.9 github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.0 github.com/ray-project/kuberay/ray-operator v0.0.0-20230920050528-dfdc51a88b6d - go.uber.org/zap v1.24.0 + go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 k8s.io/client-go v0.26.3 @@ -91,8 +91,7 @@ require ( go.opentelemetry.io/otel/sdk v1.10.0 // indirect go.opentelemetry.io/otel/trace v1.10.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect + go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.11.0 // indirect golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect diff --git a/go.sum b/go.sum index a2f26185..2c06b07d 100644 --- a/go.sum +++ b/go.sum @@ -49,7 +49,6 @@ github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrG github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -512,23 +511,23 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= From 0abc4c8f5e733536abc54394ff8428890d953176 Mon Sep 17 00:00:00 2001 From: Jiri Petrlik Date: Thu, 21 Sep 2023 10:12:23 +0200 Subject: [PATCH 139/377] Fix E2E tests README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a05315ae..699a573b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ The e2e tests can be executed locally by running the following commands: 2. Start the operator locally: ```bash - make run + NAMESPACE=default make run ``` Alternatively, You can run the operator from your IDE / debugger. From c945a368ece24e48fedcb93f5ec10fa278684baf Mon Sep 17 00:00:00 2001 From: ted chang Date: Wed, 20 Sep 2023 19:13:50 -0700 Subject: [PATCH 140/377] gsed support for MacOs --- Makefile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index ba1e4c26..facbf2cd 100644 --- a/Makefile +++ b/Makefile @@ -188,26 +188,26 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - sed -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/crd | kubectl apply -f - git restore config/* .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - sed -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - git restore config/* .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - sed -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - git restore config/* .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - sed -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - git restore config/* @@ -225,6 +225,7 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest OPENSHIFT-GOIMPORTS ?= $(LOCALBIN)/openshift-goimports OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk GH_CLI ?= $(LOCALBIN)/gh +SED ?= /usr/bin/sed ## Tool Versions KUSTOMIZE_VERSION ?= v4.5.4 @@ -279,7 +280,7 @@ validate-bundle: install-operator-sdk .PHONY: bundle bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q - sed -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion @@ -346,7 +347,7 @@ catalog-build-from-index: opm ## Build a catalog image. mkdir catalog $(OPM) render $(CATALOG_BASE_IMG) -o yaml > catalog/bundles.yaml $(OPM) render $(BUNDLE_IMG) $(OPM_BUNDLE_OPT) > catalog/codeflare-operator-bundle.yaml - sed -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml + $(SED) -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml $(OPM) validate catalog $(OPM) generate dockerfile catalog podman build . -f catalog.Dockerfile -t $(CATALOG_IMG) From c8572e17778117e64bbc18a7e4798b0c79d8e558 Mon Sep 17 00:00:00 2001 From: ted chang Date: Thu, 21 Sep 2023 09:15:37 -0700 Subject: [PATCH 141/377] Add example in README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 699a573b..ea8f5967 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,11 @@ CodeFlare Stack Compatibility Matrix Requirements: - GNU sed - sed is used in several Makefile command. Using macOS default sed is incompatible, so GNU sed is needed for correct execution of these commands. + When you have a version of the GNU sed installed on a macOS you may specify the binary using + ```bash + # brew install gnu-sed + make install -e SED=/usr/local/bin/gsed + ``` ### Testing From 1fa4979fb6519e36420acef9d3c3186a0f3f53a0 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 21 Sep 2023 12:42:28 +0200 Subject: [PATCH 142/377] Release workflow: Update Readme sed commands to process any semver value --- .github/workflows/tag-and-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 0e87a981..3fa7484d 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -85,10 +85,10 @@ jobs: - name: Adjust Compatibility Matrix in readme run: | - sed -i -E "s/(.*CodeFlare Operator.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.version }}\2${{ github.event.inputs.version }}\3/" README.md - sed -i -E "s/(.*Multi-Cluster App Dispatcher.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.mcad-version }}\2${{ github.event.inputs.mcad-version }}\3/" README.md - sed -i -E "s/(.*CodeFlare-SDK.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3/" README.md - sed -i -E "s/(.*InstaScale.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)v[0-9]+\.[0-9]+\.[0-9]+(.*)/\1${{ github.event.inputs.instascale-version }}\2${{ github.event.inputs.instascale-version }}\3/" README.md + sed -i -E "s|(.*CodeFlare Operator.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.version }}\2${{ github.event.inputs.version }}\3|" README.md + sed -i -E "s|(.*Multi-Cluster App Dispatcher.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.mcad-version }}\2${{ github.event.inputs.mcad-version }}\3|" README.md + sed -i -E "s|(.*CodeFlare-SDK.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3|" README.md + sed -i -E "s|(.*InstaScale.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.instascale-version }}\2${{ github.event.inputs.instascale-version }}\3|" README.md - name: Adjust MCAD, SDK and InstaScale dependencies in the code run: | From 65b8889ddfd11a66ea984f1fa414553ad948fc1d Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 20 Sep 2023 18:22:16 +0200 Subject: [PATCH 143/377] e2e test support: Store test pod logs and events --- test/e2e/mnist_pytorch_mcad_job_test.go | 3 - test/e2e/mnist_raycluster_sdk_test.go | 3 - test/e2e/setup.sh | 8 +- test/support/batch.go | 21 ---- test/support/core.go | 32 +++++ test/support/events.go | 148 ++++++++++++++++++++++++ test/support/ray_api.go | 2 +- test/support/test.go | 2 + 8 files changed, 186 insertions(+), 33 deletions(-) create mode 100644 test/support/events.go diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 23ad3095..adbf11bc 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -145,9 +145,6 @@ func TestMNISTPyTorchMCAD(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - // Retrieving the job logs once it has completed or timed out - defer WriteJobLogs(test, job.Namespace, job.Name) - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( Or( diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 1b59eac0..0089dcca 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -192,9 +192,6 @@ func TestMNISTRayClusterSDK(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) - // Retrieving the job logs once it has completed or timed out - defer WriteJobLogs(test, job.Namespace, job.Name) - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( Or( diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index 95a505d8..d5c99cbf 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -18,11 +18,9 @@ set -euo pipefail : "${KUBERAY_VERSION}" echo Deploying KubeRay "${KUBERAY_VERSION}" -kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=90s" +kubectl apply --server-side -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=${KUBERAY_VERSION}&timeout=180s" -kubectl create ns codeflare-system --dry-run=client -o yaml | kubectl apply -f - - -cat < maxSize { + maxSize = len(dataMap[key]) + } + } + maxStringSizeMap[key] = maxSize + } + + // Write headers + for _, header := range keys { + if _, err := content.WriteString(header); err != nil { + return nil, fmt.Errorf("error in writing the header: %v", err) + } + if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[header] - len(header) + 1)); err != nil { + return nil, fmt.Errorf("error in writing headers: %v", err) + } + if _, err := content.WriteString(" | "); err != nil { + return nil, fmt.Errorf("error in writing headers : %v", err) + } + } + if _, err := content.WriteString("\n"); err != nil { + return nil, fmt.Errorf("error in writing headers '|': %v", err) + + } + + // Write events + for _, dataMap := range dataMaps { + for _, key := range keys { + if _, err := content.WriteString(dataMap[key]); err != nil { + return nil, fmt.Errorf("error in writing events: %v", err) + } + if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[key] - len(dataMap[key]) + 1)); err != nil { + return nil, fmt.Errorf("error in writing events: %v", err) + } + if _, err := content.WriteString(" | "); err != nil { + return nil, fmt.Errorf("error in writing events: %v", err) + } + } + if _, err := content.WriteString("\n"); err != nil { + return nil, fmt.Errorf("error in writing events: %v", err) + } + } + return content.Bytes(), nil +} + +func getWhitespaceStr(size int) string { + whiteSpaceStr := "" + for i := 0; i < size; i++ { + whiteSpaceStr += " " + } + return whiteSpaceStr +} diff --git a/test/support/ray_api.go b/test/support/ray_api.go index 9004d92e..1f04f6dd 100644 --- a/test/support/ray_api.go +++ b/test/support/ray_api.go @@ -29,7 +29,7 @@ func WriteRayJobAPILogs(t Test, rayClient RayClusterClient, jobID string) { t.T().Helper() logs, err := rayClient.GetJobLogs(jobID) t.Expect(err).NotTo(gomega.HaveOccurred()) - WriteToOutputDir(t, jobID, Log, []byte(logs)) + WriteToOutputDir(t, "ray-job-log-"+jobID, Log, []byte(logs)) } func RayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) func(g gomega.Gomega) *RayJobDetailsResponse { diff --git a/test/support/test.go b/test/support/test.go index 75556ca4..683b0f21 100644 --- a/test/support/test.go +++ b/test/support/test.go @@ -131,6 +131,8 @@ func (t *T) NewTestNamespace(options ...Option[*corev1.Namespace]) *corev1.Names t.T().Helper() namespace := createTestNamespace(t, options...) t.T().Cleanup(func() { + storeAllPodLogs(t, namespace) + storeEvents(t, namespace) deleteTestNamespace(t, namespace) }) return namespace From 28ffc85d65aa4d27f3afcc01660292f86f1a9225 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 26 Sep 2023 12:04:34 +0200 Subject: [PATCH 144/377] Add missing OpenShift Config API to runtime scheme --- main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/main.go b/main.go index 9c157ebc..752ac0d7 100644 --- a/main.go +++ b/main.go @@ -48,6 +48,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/yaml" + configv1 "github.com/openshift/api/config/v1" + "github.com/project-codeflare/codeflare-operator/pkg/config" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -63,6 +65,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(mcadv1beta1.AddToScheme(scheme)) utilruntime.Must(quotasubtreev1.AddToScheme(scheme)) + utilruntime.Must(configv1.Install(scheme)) } func main() { From 99d2fc8b4e3bdcada5259f4417a8c2b2ca342430 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 26 Sep 2023 13:46:00 +0200 Subject: [PATCH 145/377] Add missing OpenShift Machine API to runtime scheme --- main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/main.go b/main.go index 752ac0d7..385a7d95 100644 --- a/main.go +++ b/main.go @@ -49,6 +49,7 @@ import ( "sigs.k8s.io/yaml" configv1 "github.com/openshift/api/config/v1" + machinev1beta1 "github.com/openshift/api/machine/v1beta1" "github.com/project-codeflare/codeflare-operator/pkg/config" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -63,9 +64,12 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + // MCAD utilruntime.Must(mcadv1beta1.AddToScheme(scheme)) utilruntime.Must(quotasubtreev1.AddToScheme(scheme)) + // InstaScale utilruntime.Must(configv1.Install(scheme)) + utilruntime.Must(machinev1beta1.Install(scheme)) } func main() { From 11fc95cf242d313228f820919da8eea9916ed51b Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Fri, 22 Sep 2023 16:54:50 +0100 Subject: [PATCH 146/377] refactor: addition of rbac needed for instascale controller --- config/rbac/instascale_role.yaml | 102 +++++++++++++++++++++++ config/rbac/instascale_role_binding.yaml | 12 +++ 2 files changed, 114 insertions(+) create mode 100644 config/rbac/instascale_role.yaml create mode 100644 config/rbac/instascale_role_binding.yaml diff --git a/config/rbac/instascale_role.yaml b/config/rbac/instascale_role.yaml new file mode 100644 index 00000000..ff7abc86 --- /dev/null +++ b/config/rbac/instascale_role.yaml @@ -0,0 +1,102 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: instascale-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update +- apiGroups: + - "" + resourceNames: + - instascale-ocm-secret + resources: + - secrets + verbs: + - get +- apiGroups: + - apps + resources: + - deployments + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - machineset + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - machineset/status + verbs: + - get +- apiGroups: + - config.openshift.io + resources: + - clusterversions + verbs: + - get + - list + - watch +- apiGroups: + - machine.openshift.io + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers/finalizers + verbs: + - update +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers/status + verbs: + - get + - patch + - update diff --git a/config/rbac/instascale_role_binding.yaml b/config/rbac/instascale_role_binding.yaml new file mode 100644 index 00000000..00a7d43f --- /dev/null +++ b/config/rbac/instascale_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: instascale-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: instascale-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system From adf83296f525d51d49b3c6a27e01807cfaebf2a4 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Mon, 25 Sep 2023 13:48:09 +0100 Subject: [PATCH 147/377] refactor: remove duplicate permissions and remove wildcard for machine.openshift.io --- config/rbac/instascale_role.yaml | 47 +++----------------------------- 1 file changed, 4 insertions(+), 43 deletions(-) diff --git a/config/rbac/instascale_role.yaml b/config/rbac/instascale_role.yaml index ff7abc86..a9063152 100644 --- a/config/rbac/instascale_role.yaml +++ b/config/rbac/instascale_role.yaml @@ -5,12 +5,6 @@ metadata: creationTimestamp: null name: instascale-role rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - apiGroups: - "" resources: @@ -22,20 +16,10 @@ rules: - update - apiGroups: - "" - resourceNames: - - instascale-ocm-secret resources: - secrets verbs: - get -- apiGroups: - - apps - resources: - - deployments - verbs: - - get - - list - - watch - apiGroups: - apps resources: @@ -65,19 +49,10 @@ rules: - apiGroups: - machine.openshift.io resources: - - '*' - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers + - controlplanemachinesets + - machinehealthchecks + - machines + - machinesets verbs: - create - delete @@ -86,17 +61,3 @@ rules: - patch - update - watch -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers/finalizers - verbs: - - update -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers/status - verbs: - - get - - patch - - update From e99201be48b5ae39bd6d99672af6811c6f090aa3 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Mon, 25 Sep 2023 17:14:25 +0100 Subject: [PATCH 148/377] refactor: limit resources for machine.openshift.io api group --- config/rbac/instascale_role.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/rbac/instascale_role.yaml b/config/rbac/instascale_role.yaml index a9063152..8c378eca 100644 --- a/config/rbac/instascale_role.yaml +++ b/config/rbac/instascale_role.yaml @@ -49,8 +49,6 @@ rules: - apiGroups: - machine.openshift.io resources: - - controlplanemachinesets - - machinehealthchecks - machines - machinesets verbs: From dafc36b9a94a6c2bafc1994a37e88558501d1637 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Tue, 26 Sep 2023 13:47:49 +0100 Subject: [PATCH 149/377] refactor: removes unneccessary permissions --- config/rbac/instascale_role.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/config/rbac/instascale_role.yaml b/config/rbac/instascale_role.yaml index 8c378eca..dec6b720 100644 --- a/config/rbac/instascale_role.yaml +++ b/config/rbac/instascale_role.yaml @@ -20,24 +20,6 @@ rules: - secrets verbs: - get -- apiGroups: - - apps - resources: - - machineset - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - machineset/status - verbs: - - get - apiGroups: - config.openshift.io resources: From c3383c8e19a8a8c94169c218f1ed8c9faa8e12d5 Mon Sep 17 00:00:00 2001 From: Jiri Petrlik Date: Tue, 12 Sep 2023 08:48:45 +0200 Subject: [PATCH 150/377] [275] Implement ExposeService method --- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 98 +------------------ test/support/ingress.go | 60 ++++++++++++ test/support/route.go | 53 ++++++++++ test/support/service.go | 27 +++++ test/support/support.go | 4 +- 5 files changed, 143 insertions(+), 99 deletions(-) create mode 100644 test/support/service.go diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 413c8c5e..1a1aab2c 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -18,7 +18,6 @@ package e2e import ( "encoding/base64" - "net/url" "testing" . "github.com/onsi/gomega" @@ -26,12 +25,8 @@ import ( rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - routev1 "github.com/openshift/api/route/v1" . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -257,98 +252,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) - var rayDashboardURL url.URL - if IsOpenShift(test) { - // Create a route to expose the Ray cluster API - route := &routev1.Route{ - TypeMeta: metav1.TypeMeta{ - APIVersion: routev1.GroupVersion.String(), - Kind: "Route", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace.Name, - Name: "ray-dashboard", - }, - Spec: routev1.RouteSpec{ - To: routev1.RouteTargetReference{ - Name: "raycluster-head-svc", - }, - Port: &routev1.RoutePort{ - TargetPort: intstr.FromString("dashboard"), - }, - }, - } - - _, err := test.Client().Route().RouteV1().Routes(namespace.Name).Create(test.Ctx(), route, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Route %s/%s successfully", route.Namespace, route.Name) - - test.T().Logf("Waiting for Route %s/%s to be admitted", route.Namespace, route.Name) - test.Eventually(Route(test, route.Namespace, route.Name), TestTimeoutMedium). - Should(WithTransform(ConditionStatus(routev1.RouteAdmitted), Equal(corev1.ConditionTrue))) - - route = GetRoute(test, route.Namespace, route.Name) - - rayDashboardURL = url.URL{ - Scheme: "http", - Host: route.Status.Ingress[0].Host, - } - } else { - ingress := &networkingv1.Ingress{ - TypeMeta: metav1.TypeMeta{ - APIVersion: networkingv1.SchemeGroupVersion.String(), - Kind: "Ingress", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace.Name, - Name: "ray-dashboard", - Annotations: map[string]string{ - "nginx.ingress.kubernetes.io/use-regex": "true", - "nginx.ingress.kubernetes.io/rewrite-target": "/$2", - }, - }, - Spec: networkingv1.IngressSpec{ - Rules: []networkingv1.IngressRule{ - { - IngressRuleValue: networkingv1.IngressRuleValue{ - HTTP: &networkingv1.HTTPIngressRuleValue{ - Paths: []networkingv1.HTTPIngressPath{ - { - Path: "/ray-dashboard(/|$)(.*)", - PathType: Ptr(networkingv1.PathTypePrefix), - Backend: networkingv1.IngressBackend{ - Service: &networkingv1.IngressServiceBackend{ - Name: "raycluster-head-svc", - Port: networkingv1.ServiceBackendPort{ - Name: "dashboard", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - _, err := test.Client().Core().NetworkingV1().Ingresses(ingress.Namespace).Create(test.Ctx(), ingress, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Ingress %s/%s successfully", ingress.Namespace, ingress.Name) - - test.T().Logf("Waiting for Ingress %s/%s to be admitted", ingress.Namespace, ingress.Name) - test.Eventually(Ingress(test, ingress.Namespace, ingress.Name), TestTimeoutMedium). - Should(WithTransform(LoadBalancerIngresses, HaveLen(1))) - - ingress = GetIngress(test, ingress.Namespace, ingress.Name) - - rayDashboardURL = url.URL{ - Scheme: "http", - Host: ingress.Status.LoadBalancer.Ingress[0].IP, - Path: "ray-dashboard", - } - } + rayDashboardURL := ExposeService(test, "ray-dashboard", namespace.Name, "raycluster-head-svc", "dashboard") test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) rayClient := NewRayClusterClient(rayDashboardURL) diff --git a/test/support/ingress.go b/test/support/ingress.go index 667abe6e..d834f3b1 100644 --- a/test/support/ingress.go +++ b/test/support/ingress.go @@ -17,6 +17,8 @@ limitations under the License. package support import ( + "net/url" + "github.com/onsi/gomega" networkingv1 "k8s.io/api/networking/v1" @@ -39,3 +41,61 @@ func GetIngress(t Test, namespace, name string) *networkingv1.Ingress { func LoadBalancerIngresses(ingress *networkingv1.Ingress) []networkingv1.IngressLoadBalancerIngress { return ingress.Status.LoadBalancer.Ingress } + +func ExposeServiceByIngress(t Test, name string, namespace string, serviceName string, servicePort string) url.URL { + ingress := &networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: networkingv1.SchemeGroupVersion.String(), + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "nginx.ingress.kubernetes.io/use-regex": "true", + "nginx.ingress.kubernetes.io/rewrite-target": "/$2", + }, + }, + Spec: networkingv1.IngressSpec{ + Rules: []networkingv1.IngressRule{ + networkingv1.IngressRule{ + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + networkingv1.HTTPIngressPath{ + Path: "/" + name + "(/|$)(.*)", + PathType: Ptr(networkingv1.PathTypePrefix), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: serviceName, + Port: networkingv1.ServiceBackendPort{ + Name: servicePort, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + _, err := t.Client().Core().NetworkingV1().Ingresses(ingress.Namespace).Create(t.Ctx(), ingress, metav1.CreateOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + t.T().Logf("Created Ingress %s/%s successfully", ingress.Namespace, ingress.Name) + + t.T().Logf("Waiting for Ingress %s/%s to be admitted", ingress.Namespace, ingress.Name) + t.Eventually(Ingress(t, ingress.Namespace, ingress.Name), TestTimeoutMedium). + Should(gomega.WithTransform(LoadBalancerIngresses, gomega.HaveLen(1))) + + ingress = GetIngress(t, ingress.Namespace, ingress.Name) + + ingressURL := url.URL{ + Scheme: "http", + Host: ingress.Status.LoadBalancer.Ingress[0].IP, + Path: name, + } + return ingressURL +} diff --git a/test/support/route.go b/test/support/route.go index 8a74ca12..df160890 100644 --- a/test/support/route.go +++ b/test/support/route.go @@ -17,9 +17,14 @@ limitations under the License. package support import ( + "net/http" + "net/url" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" routev1 "github.com/openshift/api/route/v1" ) @@ -36,3 +41,51 @@ func GetRoute(t Test, namespace, name string) *routev1.Route { t.T().Helper() return Route(t, namespace, name)(t) } + +func ExposeServiceByRoute(t Test, name string, namespace string, serviceName string, servicePort string) url.URL { + r := &routev1.Route{ + TypeMeta: metav1.TypeMeta{ + APIVersion: routev1.SchemeGroupVersion.String(), + Kind: "Route", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: routev1.RouteSpec{ + To: routev1.RouteTargetReference{ + Name: serviceName, + }, + Port: &routev1.RoutePort{ + TargetPort: intstr.FromString(servicePort), + }, + }, + } + + _, err := t.Client().Route().RouteV1().Routes(r.Namespace).Create(t.Ctx(), r, metav1.CreateOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + t.T().Logf("Created Route %s/%s successfully", r.Namespace, r.Name) + + t.T().Logf("Waiting for Route %s/%s to be available", r.Namespace, r.Name) + t.Eventually(Route(t, r.Namespace, r.Name), TestTimeoutLong). + Should(gomega.WithTransform(ConditionStatus(routev1.RouteAdmitted), gomega.Equal(corev1.ConditionTrue))) + + // Retrieve hostname + r, err = t.Client().Route().RouteV1().Routes(r.Namespace).Get(t.Ctx(), r.Name, metav1.GetOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + hostname := r.Status.Ingress[0].Host + + // Wait for expected HTTP code + t.Eventually(func() int { + resp, _ := http.Get("http://" + hostname) + return resp.StatusCode + }, TestTimeoutLong).Should(gomega.Not(gomega.Equal(503))) + + r = GetRoute(t, r.Namespace, r.Name) + routeURL := url.URL{ + Scheme: "http", + Host: r.Status.Ingress[0].Host, + } + + return routeURL +} diff --git a/test/support/service.go b/test/support/service.go new file mode 100644 index 00000000..d59ea3ec --- /dev/null +++ b/test/support/service.go @@ -0,0 +1,27 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import "net/url" + +func ExposeService(t Test, name string, namespace string, serviceName string, servicePort string) url.URL { + if IsOpenShift(t) { + return ExposeServiceByRoute(t, name, namespace, serviceName, servicePort) + } else { + return ExposeServiceByIngress(t, name, namespace, serviceName, servicePort) + } +} diff --git a/test/support/support.go b/test/support/support.go index 782e65e7..1255baa8 100644 --- a/test/support/support.go +++ b/test/support/support.go @@ -28,11 +28,11 @@ import ( ) var ( + ApplyOptions = metav1.ApplyOptions{FieldManager: "codeflare-test", Force: true} + TestTimeoutShort = 1 * time.Minute TestTimeoutMedium = 2 * time.Minute TestTimeoutLong = 5 * time.Minute - - ApplyOptions = metav1.ApplyOptions{FieldManager: "codeflare-test", Force: true} ) func init() { From 3bf1ba8cfa31872a28fac26c66914e1551fef94f Mon Sep 17 00:00:00 2001 From: Jiri Petrlik Date: Mon, 25 Sep 2023 10:57:48 +0200 Subject: [PATCH 151/377] [263] Release Action can skip intermediate steps --- .../workflows/project-codeflare-release.yml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 629ac81d..286255d1 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -41,12 +41,21 @@ jobs: runs-on: ubuntu-latest steps: + - name: Check if MCAD release does exist + run: | + status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/${{ github.event.inputs.mcad-version }}) + if [[ "$status_code" == "200" ]]; then + echo "MCAD release with version ${{ github.event.inputs.mcad-version }} already exist. Will not create MCAD release." + fi + echo "MCAD_RELEASE_STATUS_CODE=$status_code" >> $GITHUB_ENV + - name: Release MCAD run: | gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref ${{ github.ref }} --field tag=${{ github.event.inputs.mcad-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash + if: ${{ env.MCAD_RELEASE_STATUS_CODE != '200' }} - name: Wait for MCAD run to finish run: | @@ -57,18 +66,28 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash + if: ${{ env.MCAD_RELEASE_STATUS_CODE != '200' }} release-instascale: needs: release-mcad runs-on: ubuntu-latest steps: + - name: Check if Instascale release does exist + run: | + status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/project-codeflare/instascale/releases/tag/${{ github.event.inputs.instascale-version }}) + if [[ "$status_code" == "200" ]]; then + echo "Instascale release with version ${{ github.event.inputs.instascale-version }} already exist. Will not create Instascale release." + fi + echo "INSTASCALE_RELEASE_STATUS_CODE=$status_code" >> $GITHUB_ENV + - name: Release InstaScale run: | gh workflow run instascale-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field tag=${{ github.event.inputs.instascale-version }} --field mcad-version=${{ github.event.inputs.mcad-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash + if: ${{ env.INSTASCALE_RELEASE_STATUS_CODE != '200' }} - name: Wait for InstaScale run to finish run: | @@ -79,11 +98,20 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash + if: ${{ env.INSTASCALE_RELEASE_STATUS_CODE != '200' }} release-codeflare-sdk: runs-on: ubuntu-latest steps: + - name: Check if Codeflare SDK release does exist + run: | + status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/project-codeflare/codeflare-sdk/releases/tag/${{ github.event.inputs.codeflare-sdk-version }}) + if [[ "$status_code" == "200" ]]; then + echo "SDK release with version ${{ github.event.inputs.codeflare-sdk-version }} already exist. Will not create SDK release." + fi + echo "SDK_RELEASE_STATUS_CODE=$status_code" >> $GITHUB_ENV + - name: Release CodeFlare SDK run: | semver_version="${{ github.event.inputs.codeflare-sdk-version }}" @@ -92,6 +120,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash + if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} - name: Wait for CodeFlare SDK run to finish run: | @@ -102,6 +131,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash + if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} release-codeflare-operator: needs: [release-mcad, release-instascale, release-codeflare-sdk] From f3660dbbd8e9a13afe6ec9d7ca2f54d11c9750fa Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 27 Sep 2023 10:28:51 +0200 Subject: [PATCH 152/377] Include InstaScale RBAC in kustomization.yaml --- config/rbac/kustomization.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 3c96e817..a0ff39ff 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -7,6 +7,8 @@ resources: - service_account.yaml - role.yaml - role_binding.yaml +- instascale_role.yaml +- instascale_role_binding.yaml - edit_role_binding.yaml # We are using this binding as mcad requires this role - leader_election_role.yaml - leader_election_role_binding.yaml From de7fc9dce224a92282bf33665465b3a15909b55a Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Fri, 29 Sep 2023 12:50:19 +0100 Subject: [PATCH 153/377] fix: e2e tests not passing --- README.md | 5 ++++- test/e2e/mnist_pytorch_mcad_job_test.go | 18 ++++++++++++++++-- test/e2e/mnist_raycluster_sdk_test.go | 16 +++++++++++++++- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ea8f5967..75ed7d4f 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,10 @@ The e2e tests can be executed locally by running the following commands: - 'system:serviceaccount:$(namespace):kuberay-operator' ``` -4. In a separate terminal, run the e2e suite: +4. In a separate terminal, set your output directory for test files, and run the e2e suite: + ```bash + export CODEFLARE_TEST_OUTPUT_DIR= + ``` ```bash make test-e2e diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index adbf11bc..754121a3 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -77,15 +77,23 @@ func TestMNISTPyTorchMCAD(t *testing.T) { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "job", - Image: GetPyTorchImage(), + Name: "job", + Image: GetPyTorchImage(), + Env: []corev1.EnvVar{ + corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"}, + }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ { Name: "test", MountPath: "/test", }, + { + Name: "workdir", + MountPath: "/workdir", + }, }, + WorkingDir: "/workdir", }, }, Volumes: []corev1.Volume{ @@ -99,6 +107,12 @@ func TestMNISTPyTorchMCAD(t *testing.T) { }, }, }, + { + Name: "workdir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, }, RestartPolicy: corev1.RestartPolicyNever, }, diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 0089dcca..8a3bc390 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -160,14 +160,22 @@ func TestMNISTRayClusterSDK(t *testing.T) { Name: "test", // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed // See https://github.com/project-codeflare/codeflare-sdk/pull/146 - Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Env: []corev1.EnvVar{ + corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"}, + }, Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, VolumeMounts: []corev1.VolumeMount{ { Name: "test", MountPath: "/test", }, + { + Name: "workdir", + MountPath: "/workdir", + }, }, + WorkingDir: "/workdir", }, }, Volumes: []corev1.Volume{ @@ -181,6 +189,12 @@ func TestMNISTRayClusterSDK(t *testing.T) { }, }, }, + { + Name: "workdir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, }, RestartPolicy: corev1.RestartPolicyNever, ServiceAccountName: serviceAccount.Name, From a68f9c4b24165307333ae802c9d29fb3606c60fd Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 4 Oct 2023 14:40:38 +0200 Subject: [PATCH 154/377] Use 4 core runners --- .github/workflows/e2e_tests.yaml | 21 +------------------ .github/workflows/olm_tests.yaml | 21 +------------------ test/e2e/mnist_pytorch_mcad_job_test.go | 4 ++-- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 2 +- 4 files changed, 5 insertions(+), 43 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 42ee0b73..2f119bf6 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -27,28 +27,9 @@ concurrency: jobs: kubernetes-e2e: - runs-on: ubuntu-20.04 + runs-on: ubuntu-20.04-4core steps: - - name: Cleanup - run: | - ls -lart - echo "Initial status:" - df -h - - echo "Cleaning up resources:" - sudo swapoff -a - sudo rm -f /swapfile - sudo apt clean - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - docker rmi $(docker image ls -aq) - - echo "Final status:" - df -h - - name: Checkout code uses: actions/checkout@v3 with: diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 6ecede4e..5002f069 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -19,7 +19,7 @@ concurrency: jobs: kubernetes-olm-upgrade: - runs-on: ubuntu-20.04 + runs-on: ubuntu-20.04-4core timeout-minutes: 60 env: OLM_VERSION: v0.24.0 @@ -30,25 +30,6 @@ jobs: CODEFLARE_TEST_TIMEOUT_LONG: "10m" steps: - - name: Cleanup - run: | - ls -lart - echo "Initial status:" - df -h - - echo "Cleaning up resources:" - sudo swapoff -a - sudo rm -f /swapfile - sudo apt clean - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - docker rmi $(docker image ls -aq) - - echo "Final status:" - df -h - - uses: actions/checkout@v3 with: fetch-depth: 0 # fetching also previous commits to get tags diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 754121a3..84722152 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -135,11 +135,11 @@ func TestMNISTPyTorchMCAD(t *testing.T) { { Replicas: 1, Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceCPU: resource.MustParse("250m"), corev1.ResourceMemory: resource.MustParse("512Mi"), }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1G"), }, }, diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 1a1aab2c..7752d2ea 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -169,7 +169,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("512Mi"), + corev1.ResourceMemory: resource.MustParse("1G"), }, }, }, From aa4e403a8e9bc1d0f94168b95219478b1558953e Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Tue, 3 Oct 2023 12:53:57 +0100 Subject: [PATCH 155/377] fix: remove ServiceAccount binding to the edit role --- config/rbac/edit_role_binding.yaml | 12 ------------ config/rbac/kustomization.yaml | 1 - 2 files changed, 13 deletions(-) delete mode 100644 config/rbac/edit_role_binding.yaml diff --git a/config/rbac/edit_role_binding.yaml b/config/rbac/edit_role_binding.yaml deleted file mode 100644 index 640ae1ba..00000000 --- a/config/rbac/edit_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manager-edit-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: edit -subjects: - - kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index a0ff39ff..05bf4c6c 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -9,6 +9,5 @@ resources: - role_binding.yaml - instascale_role.yaml - instascale_role_binding.yaml -- edit_role_binding.yaml # We are using this binding as mcad requires this role - leader_election_role.yaml - leader_election_role_binding.yaml From 37b0253bfa7de2f4a380fbd39cf5e40e1e347e61 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Wed, 4 Oct 2023 11:16:45 +0100 Subject: [PATCH 156/377] add: batch permissions --- config/rbac/role.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 1393982c..07954b37 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -18,6 +18,17 @@ rules: - patch - update - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - list + - patch + - update + - watch - apiGroups: - apps resources: From 9ec9196ed43eb72fe6102327a67fef484483778a Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 4 Oct 2023 14:14:37 +0200 Subject: [PATCH 157/377] Change quota management API group version to v1alpha1 --- Makefile | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- main.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index facbf2cd..c7b10256 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ INSTASCALE_VERSION ?= v0.0.9 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.35.0 +MCAD_VERSION ?= 725a614debe3d34d1547c1659ef5ad49f8f6c5df MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} diff --git a/go.mod b/go.mod index c1caf36c..a45a9f18 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ require ( github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/instascale v0.0.9 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.0 - github.com/ray-project/kuberay/ray-operator v0.0.0-20230920050528-dfdc51a88b6d + github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.1-0.20231004084320-725a614debe3 + github.com/ray-project/kuberay/ray-operator v0.0.0-20231003160009-38e3527cfce7 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index 2c06b07d..572e3fb5 100644 --- a/go.sum +++ b/go.sum @@ -391,8 +391,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/instascale v0.0.9 h1:XlwprvuJNbNVnLxCa6lPz5bKUEMhJprVRumd42sRhVY= github.com/project-codeflare/instascale v0.0.9/go.mod h1:fpc8TiFvR2r0duKFbKmGTshEGa0eGK9noWWfjhPxphE= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.0 h1:7Ks2+6Jd8tnwymSmSrwNb/CpAPmgS0lNIhMlj6DrT+o= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.1-0.20231004084320-725a614debe3 h1:l/QHuyrSgaZfZ1UCRjj49rhagJV7tsyNlq0+lRhS6ZM= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.1-0.20231004084320-725a614debe3/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -420,8 +420,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230920050528-dfdc51a88b6d h1:qrCTwTnba5lQMiV6dYimR8epl8Ju90U+QLbmRQ4pe9k= -github.com/ray-project/kuberay/ray-operator v0.0.0-20230920050528-dfdc51a88b6d/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= +github.com/ray-project/kuberay/ray-operator v0.0.0-20231003160009-38e3527cfce7 h1:arS5d3gMlpVJTcBipREs6uOWP4mL7C0EtQutISpC4Mo= +github.com/ray-project/kuberay/ray-operator v0.0.0-20231003160009-38e3527cfce7/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= diff --git a/main.go b/main.go index 385a7d95..6173b853 100644 --- a/main.go +++ b/main.go @@ -27,7 +27,7 @@ import ( instascale "github.com/project-codeflare/instascale/controllers" instascaleconfig "github.com/project-codeflare/instascale/pkg/config" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - quotasubtreev1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/quotaplugins/quotasubtree/v1" + quotasubtreev1alpha1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/quotaplugins/quotasubtree/v1alpha1" mcadconfig "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config" mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob" "go.uber.org/zap/zapcore" @@ -66,7 +66,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) // MCAD utilruntime.Must(mcadv1beta1.AddToScheme(scheme)) - utilruntime.Must(quotasubtreev1.AddToScheme(scheme)) + utilruntime.Must(quotasubtreev1alpha1.AddToScheme(scheme)) // InstaScale utilruntime.Must(configv1.Install(scheme)) utilruntime.Must(machinev1beta1.Install(scheme)) From bbe1c65829bf8f6b0e64d7b82e2d084590afa22e Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 4 Sep 2023 15:16:36 +0100 Subject: [PATCH 158/377] Adding instascale e2e test --- test/e2e/instascale/instascale_test.go | 133 +++++++++++++++++++++++++ test/support/clusterpools.go | 92 +++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 test/e2e/instascale/instascale_test.go create mode 100644 test/support/clusterpools.go diff --git a/test/e2e/instascale/instascale_test.go b/test/e2e/instascale/instascale_test.go new file mode 100644 index 00000000..a4439130 --- /dev/null +++ b/test/e2e/instascale/instascale_test.go @@ -0,0 +1,133 @@ +package e2e + +import ( + "fmt" + . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-operator/test/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +var ( + ocmToken string + machinePoolsExist bool + numInitialNodePools int + numInitialMachineSets int +) + +func TestInstascale(t *testing.T) { + + test := With(t) + test.T().Parallel() + + namespace := test.NewTestNamespace() + + connection, err := CreateOCMConnection() + if err != nil { + test.T().Errorf("Unable to create ocm connection - Error : %v", err) + } + defer connection.Close() + + machinePoolsExist = true + // check existing machine pools + numInitialMachinePools, err := MachinePoolsCount(connection) + if err != nil { + test.T().Errorf("Unable to count machine pools - Error : %v", err) + } + + if numInitialMachinePools == 0 { + machinePoolsExist = false + numInitialNodePools, err = NodePoolsCount(connection) + if err != nil { + test.T().Errorf("Unable to count node pools - Error : %v", err) + } + if numInitialNodePools == 0 { + numInitialMachineSets, err = MachineSetsCount(connection) + if err != nil { + test.T().Errorf("Unable to count machine sets - Error : %v", err) + } + } + } + + // create an appwrapper + aw := &mcadv1beta1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instascale", + Namespace: namespace.Name, + Labels: map[string]string{ + "orderedinstance": "m5.xlarge_g4dn.xlarge", + }, + }, + Spec: mcadv1beta1.AppWrapperSpec{ + AggrResources: mcadv1beta1.AppWrapperResourceList{ + GenericItems: []mcadv1beta1.AppWrapperGenericResource{ + { + CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ + { + Replicas: 1, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }, + { + Replicas: 1, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }, + }, + }, + }, + }, + }, + } + + aw, err = test.Client().MCAD().McadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) + + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + if !machinePoolsExist { + numNodePools, err := NodePoolsCount(connection) + if err != nil { + test.T().Errorf("Unable to count node pools - Error : %v", err) + } + fmt.Println(numNodePools) + test.Expect(numNodePools).To(BeNumerically(">", numInitialNodePools)) + test.T().Logf("number of machine pools increased") + + } else if machinePoolsExist { + numMachinePools, err := MachinePoolsCount(connection) + if err != nil { + test.T().Errorf("Unable to count machine pools - Error : %v", err) + } + fmt.Println(numMachinePools) + test.Expect(numMachinePools).To(BeNumerically(">", numInitialMachinePools)) + test.T().Logf("number of machine pools increased") + } else { + numMachineSets, err := MachineSetsCount(connection) + if err != nil { + test.T().Errorf("Unable to count machine sets - Error : %v", err) + } + fmt.Println(numMachineSets) + test.Expect(numMachineSets).To(BeNumerically(">", numInitialMachineSets)) + test.T().Logf("number of machine sets increased") + } + + // TODO submit and check that the job has completed and that resources are released/scaled down +} diff --git a/test/support/clusterpools.go b/test/support/clusterpools.go new file mode 100644 index 00000000..7462e7a8 --- /dev/null +++ b/test/support/clusterpools.go @@ -0,0 +1,92 @@ +package support + +import ( + "context" + "fmt" + "os" + + ocmsdk "github.com/openshift-online/ocm-sdk-go" + mapiclientset "github.com/openshift/client-go/machine/clientset/versioned" + "github.com/openshift/client-go/machine/listers/machine/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + ocmToken string = os.Getenv("OCMTOKEN") + ClusterID string = os.Getenv("CLUSTERID") + machinePoolsExist bool + machineClient mapiclientset.Interface + msLister v1beta1.MachineSetLister +) + +const ( + namespaceToList = "openshift-machine-api" +) + +func CreateOCMConnection() (*ocmsdk.Connection, error) { + + logger, err := ocmsdk.NewGoLoggerBuilder(). + Debug(false). + Build() + if err != nil { + fmt.Fprintf(os.Stderr, "Can't build logger: %v\n", err) + os.Exit(1) + } + + connection, err := ocmsdk.NewConnectionBuilder(). + Logger(logger). + Tokens(ocmToken). + Build() + fmt.Println("connection", connection, err) + if err != nil || connection == nil { + fmt.Println("something went wrong", connection, err) + fmt.Fprintf(os.Stderr, "Can't build connection: %v\n", err) + os.Exit(1) + } + + return connection, nil +} + + +func MachinePoolsCount(connection *ocmsdk.Connection) (numMachinePools int, err error) { + fmt.Println("clusterID %v", ClusterID) + machinePoolsConnection := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools().List() + fmt.Println("machine pools connection %v", machinePoolsConnection) + + machinePoolsListResponse, err := machinePoolsConnection.Send() + if err != nil { + fmt.Println("machine pools list response, %v error, %v", machinePoolsListResponse, err) + return 0, fmt.Errorf("unable to send request, error: %v", err) + } + machinePoolsList := machinePoolsListResponse.Items() + fmt.Println("machine pool list %v", machinePoolsList) + //check the current number of machine pools + // TODO to be more precise could we check the machineTypes? + numMachinePools = machinePoolsList.Len() + fmt.Println(numMachinePools) + + return numMachinePools, nil +} + +func NodePoolsCount(connection *ocmsdk.Connection) (numNodePools int, err error) { + nodePoolsConnection := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).NodePools().List() + nodePoolsListResponse, err := nodePoolsConnection.SendContext(context.Background()) + if err != nil { + return 0, fmt.Errorf("unable to send request, error: %v", err) + } + nodePoolsList := nodePoolsListResponse.Items() + numNodePools = nodePoolsList.Len() + fmt.Println(numNodePools) + + return numNodePools, nil +} + +func MachineSetsCount(connection *ocmsdk.Connection) (numMachineSets int, err error) { + machineSets, err := machineClient.MachineV1beta1().MachineSets(namespaceToList).List(context.Background(), metav1.ListOptions{}) + if err != nil { + return 0, fmt.Errorf("error while listing machine sets, error: %v", err) + } + machineSetsSize := machineSets.ListMeta.Size() + + return machineSetsSize, nil +} From 1e3998382d202decbbf34704a8cf28e86c9e2b6d Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Tue, 12 Sep 2023 10:08:34 +0100 Subject: [PATCH 159/377] update to test --- test/e2e/instascale/instascale_test.go | 133 ------------- test/e2e/instascale_test.go | 264 +++++++++++++++++++++++++ test/support/clusterpools.go | 22 +-- 3 files changed, 267 insertions(+), 152 deletions(-) delete mode 100644 test/e2e/instascale/instascale_test.go create mode 100644 test/e2e/instascale_test.go diff --git a/test/e2e/instascale/instascale_test.go b/test/e2e/instascale/instascale_test.go deleted file mode 100644 index a4439130..00000000 --- a/test/e2e/instascale/instascale_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package e2e - -import ( - "fmt" - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-operator/test/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "testing" -) - -var ( - ocmToken string - machinePoolsExist bool - numInitialNodePools int - numInitialMachineSets int -) - -func TestInstascale(t *testing.T) { - - test := With(t) - test.T().Parallel() - - namespace := test.NewTestNamespace() - - connection, err := CreateOCMConnection() - if err != nil { - test.T().Errorf("Unable to create ocm connection - Error : %v", err) - } - defer connection.Close() - - machinePoolsExist = true - // check existing machine pools - numInitialMachinePools, err := MachinePoolsCount(connection) - if err != nil { - test.T().Errorf("Unable to count machine pools - Error : %v", err) - } - - if numInitialMachinePools == 0 { - machinePoolsExist = false - numInitialNodePools, err = NodePoolsCount(connection) - if err != nil { - test.T().Errorf("Unable to count node pools - Error : %v", err) - } - if numInitialNodePools == 0 { - numInitialMachineSets, err = MachineSetsCount(connection) - if err != nil { - test.T().Errorf("Unable to count machine sets - Error : %v", err) - } - } - } - - // create an appwrapper - aw := &mcadv1beta1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-instascale", - Namespace: namespace.Name, - Labels: map[string]string{ - "orderedinstance": "m5.xlarge_g4dn.xlarge", - }, - }, - Spec: mcadv1beta1.AppWrapperSpec{ - AggrResources: mcadv1beta1.AppWrapperResourceList{ - GenericItems: []mcadv1beta1.AppWrapperGenericResource{ - { - CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ - { - Replicas: 1, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("1G"), - }, - }, - { - Replicas: 1, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("1G"), - }, - }, - }, - }, - }, - }, - }, - } - - aw, err = test.Client().MCAD().McadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) - - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - - if !machinePoolsExist { - numNodePools, err := NodePoolsCount(connection) - if err != nil { - test.T().Errorf("Unable to count node pools - Error : %v", err) - } - fmt.Println(numNodePools) - test.Expect(numNodePools).To(BeNumerically(">", numInitialNodePools)) - test.T().Logf("number of machine pools increased") - - } else if machinePoolsExist { - numMachinePools, err := MachinePoolsCount(connection) - if err != nil { - test.T().Errorf("Unable to count machine pools - Error : %v", err) - } - fmt.Println(numMachinePools) - test.Expect(numMachinePools).To(BeNumerically(">", numInitialMachinePools)) - test.T().Logf("number of machine pools increased") - } else { - numMachineSets, err := MachineSetsCount(connection) - if err != nil { - test.T().Errorf("Unable to count machine sets - Error : %v", err) - } - fmt.Println(numMachineSets) - test.Expect(numMachineSets).To(BeNumerically(">", numInitialMachineSets)) - test.T().Logf("number of machine sets increased") - } - - // TODO submit and check that the job has completed and that resources are released/scaled down -} diff --git a/test/e2e/instascale_test.go b/test/e2e/instascale_test.go new file mode 100644 index 00000000..f2f26b19 --- /dev/null +++ b/test/e2e/instascale_test.go @@ -0,0 +1,264 @@ +package e2e + +import ( + "sync" + "testing" + "time" + + . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-operator/test/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + machinePoolsExist bool + numInitialNodePools int + numInitialMachineSets int + wg = &sync.WaitGroup{} +) + +func TestInstascale(t *testing.T) { + + test := With(t) + test.T().Parallel() + + namespace := test.NewTestNamespace() + + // Test configuration + config := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist-mcad", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }, + Immutable: Ptr(true), + } + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) + + // create OCM connection + instascaleOCMSecret, err := test.Client().Core().CoreV1().Secrets("default").Get(test.Ctx(), "instascale-ocm-secret", metav1.GetOptions{}) + if err != nil { + test.T().Errorf("unable to retrieve instascale-ocm-secret - Error : %v", err) + } + test.Expect(err).NotTo(HaveOccurred()) + ocmToken := string(instascaleOCMSecret.Data["token"]) + test.T().Logf("Retrieved Secret %s successfully", instascaleOCMSecret.Name) + + connection, err := CreateOCMConnection(ocmToken) + if err != nil { + test.T().Errorf("Unable to create ocm connection - Error : %v", err) + } + defer connection.Close() + + machinePoolsExist = true + // check existing cluster resources + numInitialMachinePools, err := MachinePoolsCount(connection) + if err != nil { + test.T().Errorf("Unable to count machine pools - Error : %v", err) + } + + if numInitialMachinePools == 0 { + machinePoolsExist = false + numInitialNodePools, err = NodePoolsCount(connection) + if err != nil { + test.T().Errorf("Unable to count node pools - Error : %v", err) + } + if numInitialNodePools == 0 { + numInitialMachineSets, err = MachineSetsCount() + if err != nil { + test.T().Errorf("Unable to count machine sets - Error : %v", err) + } + } + } + + // Batch Job + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "job", + Image: GetPyTorchImage(), + Env: []corev1.EnvVar{ + corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/test2"}, + }, + Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, + Args: []string{"$PYTHONUSERBASE"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + MountPath: "/test", + }, + { + Name: "test2", + MountPath: "/test2", + }, + }, + WorkingDir: "/test2", + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Name, + }, + }, + }, + }, + { + Name: "test2", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + } + + // create an appwrapper + aw := &mcadv1beta1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instascale", + Namespace: namespace.Name, + Labels: map[string]string{ + "orderedinstance": "m5.xlarge_g4dn.xlarge", + }, + }, + Spec: mcadv1beta1.AppWrapperSpec{ + AggrResources: mcadv1beta1.AppWrapperResourceList{ + GenericItems: []mcadv1beta1.AppWrapperGenericResource{ + { + CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ + { + Replicas: 1, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }, + { + Replicas: 1, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }, + }, + GenericTemplate: Raw(test, job), + }, + }, + }, + }, + } + + _, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) + + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + // wait for required resources to be created before checking them again + time.Sleep(TestTimeoutShort) + if !machinePoolsExist { + numNodePools, err := NodePoolsCount(connection) + if err != nil { + test.T().Errorf("Unable to count node pools - Error : %v", err) + } + test.Expect(numNodePools).To(BeNumerically(">", numInitialNodePools)) + test.T().Logf("number of node pools increased from %d to %d", numInitialNodePools, numNodePools) + + } else if machinePoolsExist { + numMachinePools, err := MachinePoolsCount(connection) + if err != nil { + test.T().Errorf("Unable to count machine pools - Error : %v", err) + } + test.Expect(numMachinePools).To(BeNumerically(">", numInitialMachinePools)) + test.T().Logf("number of machine pools increased from %d to %d", numInitialMachinePools, numMachinePools) + } else { + numMachineSets, err := MachineSetsCount() + if err != nil { + test.T().Errorf("Unable to count machine sets - Error : %v", err) + } + test.Expect(numMachineSets).To(BeNumerically(">", numInitialMachineSets)) + test.T().Logf("number of machine sets increased from %d to %d", numInitialMachineSets, numMachineSets) + } + + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + // Assert the job has completed successfully + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + + // AppWrapper not being updated to complete once job is finished + + time.Sleep(TestTimeoutMedium) + if !machinePoolsExist { + numNodePoolsFinal, err := NodePoolsCount(connection) + if err != nil { + test.T().Errorf("Unable to count node pools - Error : %v", err) + } + test.Expect(numNodePoolsFinal).To(BeNumerically("==", numInitialNodePools)) + test.T().Logf("number of machine pools decreased") + + } else if machinePoolsExist { + numMachinePoolsFinal, err := MachinePoolsCount(connection) + if err != nil { + test.T().Errorf("Unable to count machine pools - Error : %v", err) + } + test.Expect(numMachinePoolsFinal).To(BeNumerically("==", numInitialMachinePools)) + test.T().Logf("number of machine pools decreased") + } else { + numMachineSetsFinal, err := MachineSetsCount() + if err != nil { + test.T().Errorf("Unable to count machine sets - Error : %v", err) + } + test.Expect(numMachineSetsFinal).To(BeNumerically("==", numInitialMachineSets)) + test.T().Logf("number of machine sets decreased") + } +} diff --git a/test/support/clusterpools.go b/test/support/clusterpools.go index 7462e7a8..43d5b5bb 100644 --- a/test/support/clusterpools.go +++ b/test/support/clusterpools.go @@ -7,24 +7,19 @@ import ( ocmsdk "github.com/openshift-online/ocm-sdk-go" mapiclientset "github.com/openshift/client-go/machine/clientset/versioned" - "github.com/openshift/client-go/machine/listers/machine/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var ( - ocmToken string = os.Getenv("OCMTOKEN") ClusterID string = os.Getenv("CLUSTERID") - machinePoolsExist bool machineClient mapiclientset.Interface - msLister v1beta1.MachineSetLister ) const ( namespaceToList = "openshift-machine-api" ) -func CreateOCMConnection() (*ocmsdk.Connection, error) { - +func CreateOCMConnection(secret string) (*ocmsdk.Connection, error) { logger, err := ocmsdk.NewGoLoggerBuilder(). Debug(false). Build() @@ -32,14 +27,11 @@ func CreateOCMConnection() (*ocmsdk.Connection, error) { fmt.Fprintf(os.Stderr, "Can't build logger: %v\n", err) os.Exit(1) } - connection, err := ocmsdk.NewConnectionBuilder(). Logger(logger). - Tokens(ocmToken). + Tokens(string(secret)). Build() - fmt.Println("connection", connection, err) if err != nil || connection == nil { - fmt.Println("something went wrong", connection, err) fmt.Fprintf(os.Stderr, "Can't build connection: %v\n", err) os.Exit(1) } @@ -49,21 +41,13 @@ func CreateOCMConnection() (*ocmsdk.Connection, error) { func MachinePoolsCount(connection *ocmsdk.Connection) (numMachinePools int, err error) { - fmt.Println("clusterID %v", ClusterID) machinePoolsConnection := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools().List() - fmt.Println("machine pools connection %v", machinePoolsConnection) - machinePoolsListResponse, err := machinePoolsConnection.Send() if err != nil { - fmt.Println("machine pools list response, %v error, %v", machinePoolsListResponse, err) return 0, fmt.Errorf("unable to send request, error: %v", err) } machinePoolsList := machinePoolsListResponse.Items() - fmt.Println("machine pool list %v", machinePoolsList) - //check the current number of machine pools - // TODO to be more precise could we check the machineTypes? numMachinePools = machinePoolsList.Len() - fmt.Println(numMachinePools) return numMachinePools, nil } @@ -81,7 +65,7 @@ func NodePoolsCount(connection *ocmsdk.Connection) (numNodePools int, err error) return numNodePools, nil } -func MachineSetsCount(connection *ocmsdk.Connection) (numMachineSets int, err error) { +func MachineSetsCount() (numMachineSets int, err error) { machineSets, err := machineClient.MachineV1beta1().MachineSets(namespaceToList).List(context.Background(), metav1.ListOptions{}) if err != nil { return 0, fmt.Errorf("error while listing machine sets, error: %v", err) From 1d06ff6284d0a107f5b67c8f254f3e16bce801f2 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 22 Sep 2023 16:01:34 +0100 Subject: [PATCH 160/377] refactored to check for specific machine names --- test/e2e/instascale_test.go | 137 +++++++++++++++-------------------- test/support/clusterpools.go | 44 ++++++++--- test/support/support.go | 1 + 3 files changed, 93 insertions(+), 89 deletions(-) diff --git a/test/e2e/instascale_test.go b/test/e2e/instascale_test.go index f2f26b19..f9b6fd58 100644 --- a/test/e2e/instascale_test.go +++ b/test/e2e/instascale_test.go @@ -1,7 +1,6 @@ package e2e import ( - "sync" "testing" "time" @@ -14,13 +13,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var ( - machinePoolsExist bool - numInitialNodePools int - numInitialMachineSets int - wg = &sync.WaitGroup{} -) - func TestInstascale(t *testing.T) { test := With(t) @@ -65,25 +57,25 @@ func TestInstascale(t *testing.T) { } defer connection.Close() - machinePoolsExist = true // check existing cluster resources - numInitialMachinePools, err := MachinePoolsCount(connection) - if err != nil { - test.T().Errorf("Unable to count machine pools - Error : %v", err) - } + machinePoolsExist, err := MachinePoolsExist(connection) + test.Expect(err).NotTo(HaveOccurred()) + nodePoolsExist, err := NodePoolsExist(connection) + test.Expect(err).NotTo(HaveOccurred()) - if numInitialMachinePools == 0 { - machinePoolsExist = false - numInitialNodePools, err = NodePoolsCount(connection) - if err != nil { - test.T().Errorf("Unable to count node pools - Error : %v", err) - } - if numInitialNodePools == 0 { - numInitialMachineSets, err = MachineSetsCount() - if err != nil { - test.T().Errorf("Unable to count machine sets - Error : %v", err) - } - } + if machinePoolsExist { + // look for machine pool with aw name - expect not to find it + foundMachinePool, err := CheckMachinePools(connection, "test-instascale") + test.Expect(err).NotTo(HaveOccurred()) + test.Expect(foundMachinePool).To(BeFalse()) + } else if nodePoolsExist { + // look for node pool with aw name - expect not to find it + foundNodePool, err := CheckNodePools(connection, "test-instascale") + test.Expect(err).NotTo(HaveOccurred()) + test.Expect(foundNodePool).To(BeFalse()) + } else { + // TODO update to foundMachineSet + } // Batch Job @@ -103,13 +95,13 @@ func TestInstascale(t *testing.T) { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "job", - Image: GetPyTorchImage(), + Name: "job", + Image: GetPyTorchImage(), Env: []corev1.EnvVar{ corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/test2"}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, - Args: []string{"$PYTHONUSERBASE"}, + Args: []string{"$PYTHONUSERBASE"}, VolumeMounts: []corev1.VolumeMount{ { Name: "test", @@ -184,46 +176,41 @@ func TestInstascale(t *testing.T) { }, }, }, - GenericTemplate: Raw(test, job), + GenericTemplate: Raw(test, job), + CompletionStatus: "Complete", }, }, }, }, } - _, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) + _, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - // wait for required resources to be created before checking them again - time.Sleep(TestTimeoutShort) - if !machinePoolsExist { - numNodePools, err := NodePoolsCount(connection) - if err != nil { - test.T().Errorf("Unable to count node pools - Error : %v", err) - } - test.Expect(numNodePools).To(BeNumerically(">", numInitialNodePools)) - test.T().Logf("number of node pools increased from %d to %d", numInitialNodePools, numNodePools) - - } else if machinePoolsExist { - numMachinePools, err := MachinePoolsCount(connection) - if err != nil { - test.T().Errorf("Unable to count machine pools - Error : %v", err) - } - test.Expect(numMachinePools).To(BeNumerically(">", numInitialMachinePools)) - test.T().Logf("number of machine pools increased from %d to %d", numInitialMachinePools, numMachinePools) + // time.Sleep is used twice throughout the test, each for 30 seconds. Can look into using sync package waitGroup instead if that makes more sense + // wait for required resources to scale up before checking them again + time.Sleep(TestTimeoutThirtySeconds) + + if machinePoolsExist { + // look for machine pool with aw name - expect to find it + foundMachinePool, err := CheckMachinePools(connection, "test-instascale") + test.Expect(err).NotTo(HaveOccurred()) + test.Expect(foundMachinePool).To(BeTrue()) + } else if nodePoolsExist { + // look for node pool with aw name - expect to find it + foundNodePool, err := CheckNodePools(connection, "test-instascale") + test.Expect(err).NotTo(HaveOccurred()) + test.Expect(foundNodePool).To(BeTrue()) } else { - numMachineSets, err := MachineSetsCount() - if err != nil { - test.T().Errorf("Unable to count machine sets - Error : %v", err) - } - test.Expect(numMachineSets).To(BeNumerically(">", numInitialMachineSets)) - test.T().Logf("number of machine sets increased from %d to %d", numInitialMachineSets, numMachineSets) + // TODO update to foundMachineSet + } - + + // Assert that the job has completed test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( Or( @@ -235,30 +222,24 @@ func TestInstascale(t *testing.T) { test.Expect(GetJob(test, job.Namespace, job.Name)). To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) - // AppWrapper not being updated to complete once job is finished - - time.Sleep(TestTimeoutMedium) - if !machinePoolsExist { - numNodePoolsFinal, err := NodePoolsCount(connection) - if err != nil { - test.T().Errorf("Unable to count node pools - Error : %v", err) - } - test.Expect(numNodePoolsFinal).To(BeNumerically("==", numInitialNodePools)) - test.T().Logf("number of machine pools decreased") - - } else if machinePoolsExist { - numMachinePoolsFinal, err := MachinePoolsCount(connection) - if err != nil { - test.T().Errorf("Unable to count machine pools - Error : %v", err) - } - test.Expect(numMachinePoolsFinal).To(BeNumerically("==", numInitialMachinePools)) - test.T().Logf("number of machine pools decreased") + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) + + // allow time for the resources to scale down before checking them again + time.Sleep(TestTimeoutThirtySeconds) + + if machinePoolsExist { + // look for machine pool with aw name - expect to find it + foundMachinePool, err := CheckMachinePools(connection, "test-instascale") + test.Expect(err).NotTo(HaveOccurred()) + test.Expect(foundMachinePool).To(BeFalse()) + } else if nodePoolsExist { + // look for node pool with aw name - expect to find it + foundNodePool, err := CheckNodePools(connection, "test-instascale") + test.Expect(err).NotTo(HaveOccurred()) + test.Expect(foundNodePool).To(BeFalse()) } else { - numMachineSetsFinal, err := MachineSetsCount() - if err != nil { - test.T().Errorf("Unable to count machine sets - Error : %v", err) - } - test.Expect(numMachineSetsFinal).To(BeNumerically("==", numInitialMachineSets)) - test.T().Logf("number of machine sets decreased") + // TODO update to foundMachineSet + } } diff --git a/test/support/clusterpools.go b/test/support/clusterpools.go index 43d5b5bb..c0f45dee 100644 --- a/test/support/clusterpools.go +++ b/test/support/clusterpools.go @@ -4,15 +4,17 @@ import ( "context" "fmt" "os" + "strings" ocmsdk "github.com/openshift-online/ocm-sdk-go" + cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" mapiclientset "github.com/openshift/client-go/machine/clientset/versioned" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var ( - ClusterID string = os.Getenv("CLUSTERID") - machineClient mapiclientset.Interface + ClusterID string = os.Getenv("CLUSTERID") + machineClient mapiclientset.Interface ) const ( @@ -39,30 +41,50 @@ func CreateOCMConnection(secret string) (*ocmsdk.Connection, error) { return connection, nil } +func MachinePoolsExist(connection *ocmsdk.Connection) (bool, error) { + machinePools := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools() + return machinePools != nil, nil +} + +func NodePoolsExist(connection *ocmsdk.Connection) (bool, error) { + nodePools := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).NodePools() + return nodePools != nil, nil +} -func MachinePoolsCount(connection *ocmsdk.Connection) (numMachinePools int, err error) { +func CheckMachinePools(connection *ocmsdk.Connection, awName string) (foundMachinePool bool, err error) { machinePoolsConnection := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools().List() machinePoolsListResponse, err := machinePoolsConnection.Send() if err != nil { - return 0, fmt.Errorf("unable to send request, error: %v", err) + return false, fmt.Errorf("unable to send request, error: %v", err) } machinePoolsList := machinePoolsListResponse.Items() - numMachinePools = machinePoolsList.Len() + machinePoolsList.Range(func(index int, item *cmv1.MachinePool) bool { + instanceName, _ := item.GetID() + if strings.Contains(instanceName, awName) { + foundMachinePool = true + } + return true + }) - return numMachinePools, nil + return foundMachinePool, err } -func NodePoolsCount(connection *ocmsdk.Connection) (numNodePools int, err error) { +func CheckNodePools(connection *ocmsdk.Connection, awName string) (foundNodePool bool, err error) { nodePoolsConnection := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).NodePools().List() nodePoolsListResponse, err := nodePoolsConnection.SendContext(context.Background()) if err != nil { - return 0, fmt.Errorf("unable to send request, error: %v", err) + return false, fmt.Errorf("unable to send request, error: %v", err) } nodePoolsList := nodePoolsListResponse.Items() - numNodePools = nodePoolsList.Len() - fmt.Println(numNodePools) + nodePoolsList.Range(func(index int, item *cmv1.NodePool) bool { + instanceName, _ := item.GetID() + if strings.Contains(instanceName, awName) { + foundNodePool = true + } + return true + }) - return numNodePools, nil + return foundNodePool, err } func MachineSetsCount() (numMachineSets int, err error) { diff --git a/test/support/support.go b/test/support/support.go index 1255baa8..8e0573a0 100644 --- a/test/support/support.go +++ b/test/support/support.go @@ -31,6 +31,7 @@ var ( ApplyOptions = metav1.ApplyOptions{FieldManager: "codeflare-test", Force: true} TestTimeoutShort = 1 * time.Minute + TestTimeoutThirtySeconds = 30 * time.Second TestTimeoutMedium = 2 * time.Minute TestTimeoutLong = 5 * time.Minute ) From f66ab20627178059906068a8b117dcd7b5da9a42 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 25 Sep 2023 14:48:29 +0100 Subject: [PATCH 161/377] updating machine sets functionality --- test/e2e/instascale_test.go | 27 +++++++++++++++------------ test/support/clusterpools.go | 17 +++++++++++++++++ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/test/e2e/instascale_test.go b/test/e2e/instascale_test.go index f9b6fd58..e8e77044 100644 --- a/test/e2e/instascale_test.go +++ b/test/e2e/instascale_test.go @@ -65,17 +65,18 @@ func TestInstascale(t *testing.T) { if machinePoolsExist { // look for machine pool with aw name - expect not to find it - foundMachinePool, err := CheckMachinePools(connection, "test-instascale") + foundMachinePool, err := CheckMachinePools(connection, TestName) test.Expect(err).NotTo(HaveOccurred()) test.Expect(foundMachinePool).To(BeFalse()) } else if nodePoolsExist { // look for node pool with aw name - expect not to find it - foundNodePool, err := CheckNodePools(connection, "test-instascale") + foundNodePool, err := CheckNodePools(connection, TestName) test.Expect(err).NotTo(HaveOccurred()) test.Expect(foundNodePool).To(BeFalse()) } else { - // TODO update to foundMachineSet - + foundMachineSet, err := CheckMachineSets(TestName) + test.Expect(err).NotTo(HaveOccurred()) + test.Expect(foundMachineSet).To(BeFalse()) } // Batch Job @@ -197,17 +198,18 @@ func TestInstascale(t *testing.T) { if machinePoolsExist { // look for machine pool with aw name - expect to find it - foundMachinePool, err := CheckMachinePools(connection, "test-instascale") + foundMachinePool, err := CheckMachinePools(connection, TestName) test.Expect(err).NotTo(HaveOccurred()) test.Expect(foundMachinePool).To(BeTrue()) } else if nodePoolsExist { // look for node pool with aw name - expect to find it - foundNodePool, err := CheckNodePools(connection, "test-instascale") + foundNodePool, err := CheckNodePools(connection, TestName) test.Expect(err).NotTo(HaveOccurred()) test.Expect(foundNodePool).To(BeTrue()) } else { - // TODO update to foundMachineSet - + foundMachineSet, err := CheckMachineSets(TestName) + test.Expect(err).NotTo(HaveOccurred()) + test.Expect(foundMachineSet).To(BeTrue()) } // Assert that the job has completed @@ -230,16 +232,17 @@ func TestInstascale(t *testing.T) { if machinePoolsExist { // look for machine pool with aw name - expect to find it - foundMachinePool, err := CheckMachinePools(connection, "test-instascale") + foundMachinePool, err := CheckMachinePools(connection, TestName) test.Expect(err).NotTo(HaveOccurred()) test.Expect(foundMachinePool).To(BeFalse()) } else if nodePoolsExist { // look for node pool with aw name - expect to find it - foundNodePool, err := CheckNodePools(connection, "test-instascale") + foundNodePool, err := CheckNodePools(connection, TestName) test.Expect(err).NotTo(HaveOccurred()) test.Expect(foundNodePool).To(BeFalse()) } else { - // TODO update to foundMachineSet - + foundMachineSet, err := CheckMachineSets(TestName) + test.Expect(err).NotTo(HaveOccurred()) + test.Expect(foundMachineSet).To(BeFalse()) } } diff --git a/test/support/clusterpools.go b/test/support/clusterpools.go index c0f45dee..dc89765e 100644 --- a/test/support/clusterpools.go +++ b/test/support/clusterpools.go @@ -9,12 +9,16 @@ import ( ocmsdk "github.com/openshift-online/ocm-sdk-go" cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" mapiclientset "github.com/openshift/client-go/machine/clientset/versioned" + "github.com/openshift/client-go/machine/listers/machine/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" ) var ( ClusterID string = os.Getenv("CLUSTERID") machineClient mapiclientset.Interface + msLister v1beta1.MachineSetLister + TestName string = "test-instascale" ) const ( @@ -96,3 +100,16 @@ func MachineSetsCount() (numMachineSets int, err error) { return machineSetsSize, nil } + +func CheckMachineSets(awName string) (foundMachineSet bool, err error) { + machineSets, err := msLister.MachineSets("").List(labels.Everything()) + if err != nil { + return false, fmt.Errorf("error listing machine sets, error: %v", err) + } + for _, machineSet := range machineSets { + if strings.Contains(machineSet.Name, awName) { + foundMachineSet = true + } + } + return foundMachineSet, err +} From 0309a46a3a71043f15d64ba2a7ea67db0379afdd Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Tue, 26 Sep 2023 10:16:08 +0100 Subject: [PATCH 162/377] update vendor files --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a45a9f18..01d9e1b9 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.19 require ( github.com/onsi/gomega v1.27.10 + github.com/openshift-online/ocm-sdk-go v0.1.368 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/instascale v0.0.9 @@ -69,7 +70,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openshift-online/ocm-sdk-go v0.1.327 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect diff --git a/go.sum b/go.sum index 572e3fb5..e564c6f8 100644 --- a/go.sum +++ b/go.sum @@ -377,8 +377,8 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= -github.com/openshift-online/ocm-sdk-go v0.1.327 h1:WR822bGdQoMuZ2+dFdhZz3fpD2NlJhGr+F3FJPXvqFU= -github.com/openshift-online/ocm-sdk-go v0.1.327/go.mod h1:KYOw8kAKAHyPrJcQoVR82CneQ4ofC02Na4cXXaTq4Nw= +github.com/openshift-online/ocm-sdk-go v0.1.368 h1:qP+gkChV8WDwwpkUw1xUyjTXKdvrwyd70Gff2GMUSeU= +github.com/openshift-online/ocm-sdk-go v0.1.368/go.mod h1:KYOw8kAKAHyPrJcQoVR82CneQ4ofC02Na4cXXaTq4Nw= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= From a778de2a680dc6c135368079336fc6002c4acd24 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Wed, 27 Sep 2023 14:17:36 +0100 Subject: [PATCH 163/377] refactored test just for machine pools --- ...test.go => instascale_machinepool_test.go} | 82 ++++++------------- 1 file changed, 25 insertions(+), 57 deletions(-) rename test/e2e/{instascale_test.go => instascale_machinepool_test.go} (74%) diff --git a/test/e2e/instascale_test.go b/test/e2e/instascale_machinepool_test.go similarity index 74% rename from test/e2e/instascale_test.go rename to test/e2e/instascale_machinepool_test.go index e8e77044..44c07c81 100644 --- a/test/e2e/instascale_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -5,15 +5,17 @@ import ( "time" . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-operator/test/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/project-codeflare/codeflare-operator/test/support" ) -func TestInstascale(t *testing.T) { +func TestInstascaleMachinePool(t *testing.T) { test := With(t) test.T().Parallel() @@ -38,11 +40,12 @@ func TestInstascale(t *testing.T) { }, Immutable: Ptr(true), } + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - // create OCM connection + //create OCM connection instascaleOCMSecret, err := test.Client().Core().CoreV1().Secrets("default").Get(test.Ctx(), "instascale-ocm-secret", metav1.GetOptions{}) if err != nil { test.T().Errorf("unable to retrieve instascale-ocm-secret - Error : %v", err) @@ -57,27 +60,11 @@ func TestInstascale(t *testing.T) { } defer connection.Close() - // check existing cluster resources - machinePoolsExist, err := MachinePoolsExist(connection) - test.Expect(err).NotTo(HaveOccurred()) - nodePoolsExist, err := NodePoolsExist(connection) + // check existing cluster machine pool resources + // look for machine pool with aw name - expect not to find it + foundMachinePool, err := CheckMachinePools(connection, TestName) test.Expect(err).NotTo(HaveOccurred()) - - if machinePoolsExist { - // look for machine pool with aw name - expect not to find it - foundMachinePool, err := CheckMachinePools(connection, TestName) - test.Expect(err).NotTo(HaveOccurred()) - test.Expect(foundMachinePool).To(BeFalse()) - } else if nodePoolsExist { - // look for node pool with aw name - expect not to find it - foundNodePool, err := CheckNodePools(connection, TestName) - test.Expect(err).NotTo(HaveOccurred()) - test.Expect(foundNodePool).To(BeFalse()) - } else { - foundMachineSet, err := CheckMachineSets(TestName) - test.Expect(err).NotTo(HaveOccurred()) - test.Expect(foundMachineSet).To(BeFalse()) - } + test.Expect(foundMachinePool).To(BeFalse()) // Batch Job job := &batchv1.Job{ @@ -159,10 +146,12 @@ func TestInstascale(t *testing.T) { Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("250m"), corev1.ResourceMemory: resource.MustParse("512Mi"), + "nvidia.com/gpu": resource.MustParse("1"), }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("1G"), + "nvidia.com/gpu": resource.MustParse("1"), }, }, { @@ -194,23 +183,12 @@ func TestInstascale(t *testing.T) { // time.Sleep is used twice throughout the test, each for 30 seconds. Can look into using sync package waitGroup instead if that makes more sense // wait for required resources to scale up before checking them again - time.Sleep(TestTimeoutThirtySeconds) - - if machinePoolsExist { - // look for machine pool with aw name - expect to find it - foundMachinePool, err := CheckMachinePools(connection, TestName) - test.Expect(err).NotTo(HaveOccurred()) - test.Expect(foundMachinePool).To(BeTrue()) - } else if nodePoolsExist { - // look for node pool with aw name - expect to find it - foundNodePool, err := CheckNodePools(connection, TestName) - test.Expect(err).NotTo(HaveOccurred()) - test.Expect(foundNodePool).To(BeTrue()) - } else { - foundMachineSet, err := CheckMachineSets(TestName) - test.Expect(err).NotTo(HaveOccurred()) - test.Expect(foundMachineSet).To(BeTrue()) - } + time.Sleep(TestTimeoutMedium) + + // look for machine pool with aw name - expect to find it + foundMachinePool, err = CheckMachinePools(connection, TestName) + test.Expect(err).NotTo(HaveOccurred()) + test.Expect(foundMachinePool).To(BeTrue()) // Assert that the job has completed test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) @@ -228,21 +206,11 @@ func TestInstascale(t *testing.T) { Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) // allow time for the resources to scale down before checking them again - time.Sleep(TestTimeoutThirtySeconds) - - if machinePoolsExist { - // look for machine pool with aw name - expect to find it - foundMachinePool, err := CheckMachinePools(connection, TestName) - test.Expect(err).NotTo(HaveOccurred()) - test.Expect(foundMachinePool).To(BeFalse()) - } else if nodePoolsExist { - // look for node pool with aw name - expect to find it - foundNodePool, err := CheckNodePools(connection, TestName) - test.Expect(err).NotTo(HaveOccurred()) - test.Expect(foundNodePool).To(BeFalse()) - } else { - foundMachineSet, err := CheckMachineSets(TestName) - test.Expect(err).NotTo(HaveOccurred()) - test.Expect(foundMachineSet).To(BeFalse()) - } + time.Sleep(TestTimeoutMedium) + + // look for machine pool with aw name - expect not to find it + foundMachinePool, err = CheckMachinePools(connection, TestName) + test.Expect(err).NotTo(HaveOccurred()) + test.Expect(foundMachinePool).To(BeFalse()) + } From 3ff5cac1f5fa949839e6f0ac6cde5e7f94cf7c90 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 28 Sep 2023 10:12:15 +0100 Subject: [PATCH 164/377] moving common instascale test functions to separate file --- test/e2e/instascale.go | 174 ++++++++++++++++++++++++ test/e2e/instascale_machinepool_test.go | 157 +-------------------- 2 files changed, 181 insertions(+), 150 deletions(-) create mode 100644 test/e2e/instascale.go diff --git a/test/e2e/instascale.go b/test/e2e/instascale.go new file mode 100644 index 00000000..aa143f6b --- /dev/null +++ b/test/e2e/instascale.go @@ -0,0 +1,174 @@ +package e2e + +import ( + . "github.com/onsi/gomega" + ocmsdk "github.com/openshift-online/ocm-sdk-go" + . "github.com/project-codeflare/codeflare-operator/test/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestConfig(test Test, namespace string) (*corev1.ConfigMap, error) { + // Test configuration + configMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist-mcad", + Namespace: namespace, + }, + BinaryData: map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }, + Immutable: Ptr(true), + } + + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace).Create(test.Ctx(), configMap, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) + + return configMap, err +} + +func CreateConnection(test Test) (*ocmsdk.Connection, error) { + instascaleOCMSecret, err := test.Client().Core().CoreV1().Secrets("default").Get(test.Ctx(), "instascale-ocm-secret", metav1.GetOptions{}) + if err != nil { + test.T().Errorf("unable to retrieve instascale-ocm-secret - Error : %v", err) + } + test.Expect(err).NotTo(HaveOccurred()) + ocmToken := string(instascaleOCMSecret.Data["token"]) + test.T().Logf("Retrieved Secret %s successfully", instascaleOCMSecret.Name) + + connection, err := CreateOCMConnection(ocmToken) + if err != nil { + test.T().Errorf("Unable to create ocm connection - Error : %v", err) + } + return connection, err +} + +func JobAppwrapperSetup(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) (*batchv1.Job, *mcadv1beta1.AppWrapper, error) { + // Batch Job + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist", + Namespace: namespace.Name, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "job", + Image: GetPyTorchImage(), + Env: []corev1.EnvVar{ + corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/test2"}, + }, + Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, + Args: []string{"$PYTHONUSERBASE"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + MountPath: "/test", + }, + { + Name: "test2", + MountPath: "/test2", + }, + }, + WorkingDir: "/test2", + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Name, + }, + }, + }, + }, + { + Name: "test2", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + } + + // create an appwrapper + aw := &mcadv1beta1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instascale", + Namespace: namespace.Name, + Labels: map[string]string{ + "orderedinstance": "m5.xlarge_g4dn.xlarge", + }, + }, + Spec: mcadv1beta1.AppWrapperSpec{ + AggrResources: mcadv1beta1.AppWrapperResourceList{ + GenericItems: []mcadv1beta1.AppWrapperGenericResource{ + { + CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ + { + Replicas: 1, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + "nvidia.com/gpu": resource.MustParse("1"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1G"), + "nvidia.com/gpu": resource.MustParse("1"), + }, + }, + { + Replicas: 1, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }, + }, + GenericTemplate: Raw(test, job), + CompletionStatus: "Complete", + }, + }, + }, + }, + } + + _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) + + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + return job, aw, err +} diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 44c07c81..4e5ece24 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -6,12 +6,8 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -23,41 +19,13 @@ func TestInstascaleMachinePool(t *testing.T) { namespace := test.NewTestNamespace() // Test configuration - config := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist-mcad", - Namespace: namespace.Name, - }, - BinaryData: map[string][]byte{ - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }, - Immutable: Ptr(true), - } - - config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) + config, err := TestConfig(test, namespace.Name) + test.Expect(err).To(BeNil()) //create OCM connection - instascaleOCMSecret, err := test.Client().Core().CoreV1().Secrets("default").Get(test.Ctx(), "instascale-ocm-secret", metav1.GetOptions{}) - if err != nil { - test.T().Errorf("unable to retrieve instascale-ocm-secret - Error : %v", err) - } - test.Expect(err).NotTo(HaveOccurred()) - ocmToken := string(instascaleOCMSecret.Data["token"]) - test.T().Logf("Retrieved Secret %s successfully", instascaleOCMSecret.Name) + connection, err := CreateConnection(test) + test.Expect(err).To(BeNil()) - connection, err := CreateOCMConnection(ocmToken) - if err != nil { - test.T().Errorf("Unable to create ocm connection - Error : %v", err) - } defer connection.Close() // check existing cluster machine pool resources @@ -66,120 +34,9 @@ func TestInstascaleMachinePool(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.Expect(foundMachinePool).To(BeFalse()) - // Batch Job - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist", - Namespace: namespace.Name, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "job", - Image: GetPyTorchImage(), - Env: []corev1.EnvVar{ - corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/test2"}, - }, - Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, - Args: []string{"$PYTHONUSERBASE"}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - { - Name: "test2", - MountPath: "/test2", - }, - }, - WorkingDir: "/test2", - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - { - Name: "test2", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - }, - }, - } - - // create an appwrapper - aw := &mcadv1beta1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-instascale", - Namespace: namespace.Name, - Labels: map[string]string{ - "orderedinstance": "m5.xlarge_g4dn.xlarge", - }, - }, - Spec: mcadv1beta1.AppWrapperSpec{ - AggrResources: mcadv1beta1.AppWrapperResourceList{ - GenericItems: []mcadv1beta1.AppWrapperGenericResource{ - { - CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ - { - Replicas: 1, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - "nvidia.com/gpu": resource.MustParse("1"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("1G"), - "nvidia.com/gpu": resource.MustParse("1"), - }, - }, - { - Replicas: 1, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("1G"), - }, - }, - }, - GenericTemplate: Raw(test, job), - CompletionStatus: "Complete", - }, - }, - }, - }, - } - - _, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) - - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + // Setup batch job and AppWrapper + job, aw, err := JobAppwrapperSetup(test, namespace, config) + test.Expect(err).To(BeNil()) // time.Sleep is used twice throughout the test, each for 30 seconds. Can look into using sync package waitGroup instead if that makes more sense // wait for required resources to scale up before checking them again From 843b2901443476c52ff0e1972166c611c3d9d974 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 2 Oct 2023 10:21:27 +0100 Subject: [PATCH 165/377] addressing feedback --- test/e2e/instascale.go | 8 ++--- test/e2e/instascale_machinepool_test.go | 6 ++-- test/support/clusterpools.go | 39 ++----------------------- test/support/support.go | 1 - 4 files changed, 9 insertions(+), 45 deletions(-) diff --git a/test/e2e/instascale.go b/test/e2e/instascale.go index aa143f6b..5c70576e 100644 --- a/test/e2e/instascale.go +++ b/test/e2e/instascale.go @@ -85,11 +85,11 @@ func JobAppwrapperSetup(test Test, namespace *corev1.Namespace, config *corev1.C MountPath: "/test", }, { - Name: "test2", - MountPath: "/test2", + Name: "workdir", + MountPath: "/workdir", }, }, - WorkingDir: "/test2", + WorkingDir: "workdir", }, }, Volumes: []corev1.Volume{ @@ -104,7 +104,7 @@ func JobAppwrapperSetup(test Test, namespace *corev1.Namespace, config *corev1.C }, }, { - Name: "test2", + Name: "workdir", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 4e5ece24..b60d583e 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -20,11 +20,11 @@ func TestInstascaleMachinePool(t *testing.T) { // Test configuration config, err := TestConfig(test, namespace.Name) - test.Expect(err).To(BeNil()) + test.Expect(err).NotTo(HaveOccurred()) //create OCM connection connection, err := CreateConnection(test) - test.Expect(err).To(BeNil()) + test.Expect(err).NotTo(HaveOccurred()) defer connection.Close() @@ -36,7 +36,7 @@ func TestInstascaleMachinePool(t *testing.T) { // Setup batch job and AppWrapper job, aw, err := JobAppwrapperSetup(test, namespace, config) - test.Expect(err).To(BeNil()) + test.Expect(err).NotTo(HaveOccurred()) // time.Sleep is used twice throughout the test, each for 30 seconds. Can look into using sync package waitGroup instead if that makes more sense // wait for required resources to scale up before checking them again diff --git a/test/support/clusterpools.go b/test/support/clusterpools.go index dc89765e..56db165a 100644 --- a/test/support/clusterpools.go +++ b/test/support/clusterpools.go @@ -10,8 +10,6 @@ import ( cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" mapiclientset "github.com/openshift/client-go/machine/clientset/versioned" "github.com/openshift/client-go/machine/listers/machine/v1beta1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" ) var ( @@ -31,7 +29,7 @@ func CreateOCMConnection(secret string) (*ocmsdk.Connection, error) { Build() if err != nil { fmt.Fprintf(os.Stderr, "Can't build logger: %v\n", err) - os.Exit(1) + return nil, err } connection, err := ocmsdk.NewConnectionBuilder(). Logger(logger). @@ -39,22 +37,12 @@ func CreateOCMConnection(secret string) (*ocmsdk.Connection, error) { Build() if err != nil || connection == nil { fmt.Fprintf(os.Stderr, "Can't build connection: %v\n", err) - os.Exit(1) + return nil, err } return connection, nil } -func MachinePoolsExist(connection *ocmsdk.Connection) (bool, error) { - machinePools := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools() - return machinePools != nil, nil -} - -func NodePoolsExist(connection *ocmsdk.Connection) (bool, error) { - nodePools := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).NodePools() - return nodePools != nil, nil -} - func CheckMachinePools(connection *ocmsdk.Connection, awName string) (foundMachinePool bool, err error) { machinePoolsConnection := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools().List() machinePoolsListResponse, err := machinePoolsConnection.Send() @@ -90,26 +78,3 @@ func CheckNodePools(connection *ocmsdk.Connection, awName string) (foundNodePool return foundNodePool, err } - -func MachineSetsCount() (numMachineSets int, err error) { - machineSets, err := machineClient.MachineV1beta1().MachineSets(namespaceToList).List(context.Background(), metav1.ListOptions{}) - if err != nil { - return 0, fmt.Errorf("error while listing machine sets, error: %v", err) - } - machineSetsSize := machineSets.ListMeta.Size() - - return machineSetsSize, nil -} - -func CheckMachineSets(awName string) (foundMachineSet bool, err error) { - machineSets, err := msLister.MachineSets("").List(labels.Everything()) - if err != nil { - return false, fmt.Errorf("error listing machine sets, error: %v", err) - } - for _, machineSet := range machineSets { - if strings.Contains(machineSet.Name, awName) { - foundMachineSet = true - } - } - return foundMachineSet, err -} diff --git a/test/support/support.go b/test/support/support.go index 8e0573a0..1255baa8 100644 --- a/test/support/support.go +++ b/test/support/support.go @@ -31,7 +31,6 @@ var ( ApplyOptions = metav1.ApplyOptions{FieldManager: "codeflare-test", Force: true} TestTimeoutShort = 1 * time.Minute - TestTimeoutThirtySeconds = 30 * time.Second TestTimeoutMedium = 2 * time.Minute TestTimeoutLong = 5 * time.Minute ) From a975c0f9977e60103a77d66eb42c97e8702c07ed Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 2 Oct 2023 10:29:06 +0100 Subject: [PATCH 166/377] fixing PR checks --- test/e2e/instascale.go | 7 +++++-- test/e2e/instascale_machinepool_test.go | 2 ++ test/support/clusterpools.go | 12 ++---------- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/test/e2e/instascale.go b/test/e2e/instascale.go index 5c70576e..3fccf0d5 100644 --- a/test/e2e/instascale.go +++ b/test/e2e/instascale.go @@ -2,13 +2,16 @@ package e2e import ( . "github.com/onsi/gomega" - ocmsdk "github.com/openshift-online/ocm-sdk-go" - . "github.com/project-codeflare/codeflare-operator/test/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + ocmsdk "github.com/openshift-online/ocm-sdk-go" + + . "github.com/project-codeflare/codeflare-operator/test/support" ) func TestConfig(test Test, namespace string) (*corev1.ConfigMap, error) { diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index b60d583e..3a756dd7 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -6,8 +6,10 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + . "github.com/project-codeflare/codeflare-operator/test/support" ) diff --git a/test/support/clusterpools.go b/test/support/clusterpools.go index 56db165a..9a389aea 100644 --- a/test/support/clusterpools.go +++ b/test/support/clusterpools.go @@ -8,19 +8,11 @@ import ( ocmsdk "github.com/openshift-online/ocm-sdk-go" cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" - mapiclientset "github.com/openshift/client-go/machine/clientset/versioned" - "github.com/openshift/client-go/machine/listers/machine/v1beta1" ) var ( - ClusterID string = os.Getenv("CLUSTERID") - machineClient mapiclientset.Interface - msLister v1beta1.MachineSetLister - TestName string = "test-instascale" -) - -const ( - namespaceToList = "openshift-machine-api" + ClusterID string = os.Getenv("CLUSTERID") + TestName string = "test-instascale" ) func CreateOCMConnection(secret string) (*ocmsdk.Connection, error) { From 1fe7f1626ab50c4bcf05cc073619a39a3e26ab3b Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 2 Oct 2023 13:40:02 +0200 Subject: [PATCH 167/377] Refactor Machine pool functions --- ...nstascale.go => instascale_app_wrapper.go} | 55 ++----------------- test/e2e/instascale_machinepool_test.go | 37 +++++-------- test/support/clusterpools.go | 44 ++++----------- test/support/codeflare.go | 14 ++++- test/support/config_map.go | 44 +++++++++++++++ test/support/ocm.go | 50 +++++++++++++++++ test/support/support.go | 7 ++- 7 files changed, 143 insertions(+), 108 deletions(-) rename test/e2e/{instascale.go => instascale_app_wrapper.go} (66%) create mode 100644 test/support/config_map.go create mode 100644 test/support/ocm.go diff --git a/test/e2e/instascale.go b/test/e2e/instascale_app_wrapper.go similarity index 66% rename from test/e2e/instascale.go rename to test/e2e/instascale_app_wrapper.go index 3fccf0d5..d5f14efe 100644 --- a/test/e2e/instascale.go +++ b/test/e2e/instascale_app_wrapper.go @@ -9,55 +9,10 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ocmsdk "github.com/openshift-online/ocm-sdk-go" - . "github.com/project-codeflare/codeflare-operator/test/support" ) -func TestConfig(test Test, namespace string) (*corev1.ConfigMap, error) { - // Test configuration - configMap := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist-mcad", - Namespace: namespace, - }, - BinaryData: map[string][]byte{ - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }, - Immutable: Ptr(true), - } - - config, err := test.Client().Core().CoreV1().ConfigMaps(namespace).Create(test.Ctx(), configMap, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - - return configMap, err -} - -func CreateConnection(test Test) (*ocmsdk.Connection, error) { - instascaleOCMSecret, err := test.Client().Core().CoreV1().Secrets("default").Get(test.Ctx(), "instascale-ocm-secret", metav1.GetOptions{}) - if err != nil { - test.T().Errorf("unable to retrieve instascale-ocm-secret - Error : %v", err) - } - test.Expect(err).NotTo(HaveOccurred()) - ocmToken := string(instascaleOCMSecret.Data["token"]) - test.T().Logf("Retrieved Secret %s successfully", instascaleOCMSecret.Name) - - connection, err := CreateOCMConnection(ocmToken) - if err != nil { - test.T().Errorf("Unable to create ocm connection - Error : %v", err) - } - return connection, err -} - -func JobAppwrapperSetup(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) (*batchv1.Job, *mcadv1beta1.AppWrapper, error) { +func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) (*batchv1.Job, *mcadv1beta1.AppWrapper, error) { // Batch Job job := &batchv1.Job{ TypeMeta: metav1.TypeMeta{ @@ -78,7 +33,7 @@ func JobAppwrapperSetup(test Test, namespace *corev1.Namespace, config *corev1.C Name: "job", Image: GetPyTorchImage(), Env: []corev1.EnvVar{ - corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/test2"}, + {Name: "PYTHONUSERBASE", Value: "/workdir"}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, Args: []string{"$PYTHONUSERBASE"}, @@ -92,7 +47,7 @@ func JobAppwrapperSetup(test Test, namespace *corev1.Namespace, config *corev1.C MountPath: "/workdir", }, }, - WorkingDir: "workdir", + WorkingDir: "/workdir", }, }, Volumes: []corev1.Volume{ @@ -125,7 +80,7 @@ func JobAppwrapperSetup(test Test, namespace *corev1.Namespace, config *corev1.C Name: "test-instascale", Namespace: namespace.Name, Labels: map[string]string{ - "orderedinstance": "m5.xlarge_g4dn.xlarge", + "orderedinstance": "g4dn.xlarge", }, }, Spec: mcadv1beta1.AppWrapperSpec{ @@ -170,7 +125,7 @@ func JobAppwrapperSetup(test Test, namespace *corev1.Namespace, config *corev1.C test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) return job, aw, err diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 3a756dd7..ad4143cf 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -2,7 +2,6 @@ package e2e import ( "testing" - "time" . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" @@ -21,33 +20,31 @@ func TestInstascaleMachinePool(t *testing.T) { namespace := test.NewTestNamespace() // Test configuration - config, err := TestConfig(test, namespace.Name) - test.Expect(err).NotTo(HaveOccurred()) + testConfigData := map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + } + cm := CreateConfigMap(test, namespace.Name, testConfigData) //create OCM connection - connection, err := CreateConnection(test) - test.Expect(err).NotTo(HaveOccurred()) + connection := CreateOCMConnection(test) defer connection.Close() // check existing cluster machine pool resources // look for machine pool with aw name - expect not to find it - foundMachinePool, err := CheckMachinePools(connection, TestName) - test.Expect(err).NotTo(HaveOccurred()) - test.Expect(foundMachinePool).To(BeFalse()) + test.Expect(GetMachinePools(test, connection)). + ShouldNot(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) // Setup batch job and AppWrapper - job, aw, err := JobAppwrapperSetup(test, namespace, config) + job, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) test.Expect(err).NotTo(HaveOccurred()) - // time.Sleep is used twice throughout the test, each for 30 seconds. Can look into using sync package waitGroup instead if that makes more sense - // wait for required resources to scale up before checking them again - time.Sleep(TestTimeoutMedium) - // look for machine pool with aw name - expect to find it - foundMachinePool, err = CheckMachinePools(connection, TestName) - test.Expect(err).NotTo(HaveOccurred()) - test.Expect(foundMachinePool).To(BeTrue()) + test.Eventually(MachinePools(test, connection), TestTimeoutLong). + Should(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) // Assert that the job has completed test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) @@ -64,12 +61,8 @@ func TestInstascaleMachinePool(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - // allow time for the resources to scale down before checking them again - time.Sleep(TestTimeoutMedium) - // look for machine pool with aw name - expect not to find it - foundMachinePool, err = CheckMachinePools(connection, TestName) - test.Expect(err).NotTo(HaveOccurred()) - test.Expect(foundMachinePool).To(BeFalse()) + test.Eventually(MachinePools(test, connection), TestTimeoutLong). + ShouldNot(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) } diff --git a/test/support/clusterpools.go b/test/support/clusterpools.go index 9a389aea..1bd1036f 100644 --- a/test/support/clusterpools.go +++ b/test/support/clusterpools.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/onsi/gomega" ocmsdk "github.com/openshift-online/ocm-sdk-go" cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" ) @@ -15,42 +16,21 @@ var ( TestName string = "test-instascale" ) -func CreateOCMConnection(secret string) (*ocmsdk.Connection, error) { - logger, err := ocmsdk.NewGoLoggerBuilder(). - Debug(false). - Build() - if err != nil { - fmt.Fprintf(os.Stderr, "Can't build logger: %v\n", err) - return nil, err - } - connection, err := ocmsdk.NewConnectionBuilder(). - Logger(logger). - Tokens(string(secret)). - Build() - if err != nil || connection == nil { - fmt.Fprintf(os.Stderr, "Can't build connection: %v\n", err) - return nil, err +func MachinePools(t Test, connection *ocmsdk.Connection) func(g gomega.Gomega) []*cmv1.MachinePool { + return func(g gomega.Gomega) []*cmv1.MachinePool { + machinePoolsListResponse, err := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools().List().Send() + g.Expect(err).NotTo(gomega.HaveOccurred()) + return machinePoolsListResponse.Items().Slice() } - - return connection, nil } -func CheckMachinePools(connection *ocmsdk.Connection, awName string) (foundMachinePool bool, err error) { - machinePoolsConnection := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools().List() - machinePoolsListResponse, err := machinePoolsConnection.Send() - if err != nil { - return false, fmt.Errorf("unable to send request, error: %v", err) - } - machinePoolsList := machinePoolsListResponse.Items() - machinePoolsList.Range(func(index int, item *cmv1.MachinePool) bool { - instanceName, _ := item.GetID() - if strings.Contains(instanceName, awName) { - foundMachinePool = true - } - return true - }) +func GetMachinePools(t Test, connection *ocmsdk.Connection) []*cmv1.MachinePool { + t.T().Helper() + return MachinePools(t, connection)(t) +} - return foundMachinePool, err +func MachinePoolId(machinePool *cmv1.MachinePool) string { + return machinePool.ID() } func CheckNodePools(connection *ocmsdk.Connection, awName string) (foundNodePool bool, err error) { diff --git a/test/support/codeflare.go b/test/support/codeflare.go index 04b1f3e9..c378c4e6 100644 --- a/test/support/codeflare.go +++ b/test/support/codeflare.go @@ -30,8 +30,12 @@ const ( CodeFlareTestPyTorchImage = "CODEFLARE_TEST_PYTORCH_IMAGE" // The testing output directory, to write output files into. - CodeFlareTestOutputDir = "CODEFLARE_TEST_OUTPUT_DIR" + + // The name of a secret containing InstaScale OCM token. + InstaScaleOcmSecretName = "INSTASCALE_OCM_SECRET_NAME" + // The namespace where a secret containing InstaScale OCM token is stored. + InstaScaleOcmSecretNamespace = "INSTASCALE_OCM_SECRET_NAMESPACE" ) func GetCodeFlareSDKVersion() string { @@ -50,6 +54,14 @@ func GetPyTorchImage() string { return lookupEnvOrDefault(CodeFlareTestPyTorchImage, "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime") } +func GetInstaScaleOcmSecretName() string { + return lookupEnvOrDefault(InstaScaleOcmSecretName, "instascale-ocm-secret") +} + +func GetInstaScaleOcmSecretNamespace() string { + return lookupEnvOrDefault(InstaScaleOcmSecretNamespace, "default") +} + func lookupEnvOrDefault(key, value string) string { if v, ok := os.LookupEnv(key); ok { return v diff --git a/test/support/config_map.go b/test/support/config_map.go new file mode 100644 index 00000000..8846bec2 --- /dev/null +++ b/test/support/config_map.go @@ -0,0 +1,44 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func CreateConfigMap(t Test, namespace string, content map[string][]byte) *corev1.ConfigMap { + configMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "config-", + Namespace: namespace, + }, + BinaryData: content, + Immutable: Ptr(true), + } + + configMap, err := t.Client().Core().CoreV1().ConfigMaps(namespace).Create(t.Ctx(), configMap, metav1.CreateOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + t.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) + + return configMap +} diff --git a/test/support/ocm.go b/test/support/ocm.go new file mode 100644 index 00000000..235143ba --- /dev/null +++ b/test/support/ocm.go @@ -0,0 +1,50 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "fmt" + "os" + + "github.com/onsi/gomega" + ocmsdk "github.com/openshift-online/ocm-sdk-go" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func CreateOCMConnection(test Test) *ocmsdk.Connection { + instascaleOCMSecret, err := test.Client().Core().CoreV1().Secrets(GetInstaScaleOcmSecretNamespace()).Get(test.Ctx(), GetInstaScaleOcmSecretName(), metav1.GetOptions{}) + test.Expect(err).NotTo(gomega.HaveOccurred()) + + ocmToken := string(instascaleOCMSecret.Data["token"]) + test.T().Logf("Retrieved Secret %s/%s successfully", instascaleOCMSecret.Namespace, instascaleOCMSecret.Name) + + connection, err := buildOCMConnection(ocmToken) + test.Expect(err).NotTo(gomega.HaveOccurred()) + return connection +} + +func buildOCMConnection(secret string) (*ocmsdk.Connection, error) { + connection, err := ocmsdk.NewConnectionBuilder(). + Tokens(secret). + Build() + if err != nil || connection == nil { + fmt.Fprintf(os.Stderr, "Can't build connection: %v\n", err) + return nil, err + } + + return connection, nil +} diff --git a/test/support/support.go b/test/support/support.go index 1255baa8..0091b3bf 100644 --- a/test/support/support.go +++ b/test/support/support.go @@ -30,9 +30,10 @@ import ( var ( ApplyOptions = metav1.ApplyOptions{FieldManager: "codeflare-test", Force: true} - TestTimeoutShort = 1 * time.Minute - TestTimeoutMedium = 2 * time.Minute - TestTimeoutLong = 5 * time.Minute + TestTimeoutShort = 1 * time.Minute + TestTimeoutMedium = 2 * time.Minute + TestTimeoutLong = 5 * time.Minute + TestTimeoutGpuProvisioning = 30 * time.Minute ) func init() { From ac34d6782b1dad8a2cafc00b7531f6c6989bcd23 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 3 Oct 2023 10:08:05 +0200 Subject: [PATCH 168/377] Move OSD cluster ID into codeflare.go --- test/support/clusterpools.go | 34 +++++----------------------------- test/support/codeflare.go | 6 ++++++ test/support/config_map.go | 1 + test/support/ocm.go | 4 +++- 4 files changed, 15 insertions(+), 30 deletions(-) diff --git a/test/support/clusterpools.go b/test/support/clusterpools.go index 1bd1036f..9b1f7848 100644 --- a/test/support/clusterpools.go +++ b/test/support/clusterpools.go @@ -1,24 +1,18 @@ package support import ( - "context" - "fmt" - "os" - "strings" - "github.com/onsi/gomega" + ocmsdk "github.com/openshift-online/ocm-sdk-go" cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" ) -var ( - ClusterID string = os.Getenv("CLUSTERID") - TestName string = "test-instascale" -) - func MachinePools(t Test, connection *ocmsdk.Connection) func(g gomega.Gomega) []*cmv1.MachinePool { + osdClusterId, found := GetOsdClusterId() + t.Expect(found).To(gomega.BeTrue(), "OSD cluster id not found, please configure environment properly") + return func(g gomega.Gomega) []*cmv1.MachinePool { - machinePoolsListResponse, err := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools().List().Send() + machinePoolsListResponse, err := connection.ClustersMgmt().V1().Clusters().Cluster(osdClusterId).MachinePools().List().Send() g.Expect(err).NotTo(gomega.HaveOccurred()) return machinePoolsListResponse.Items().Slice() } @@ -32,21 +26,3 @@ func GetMachinePools(t Test, connection *ocmsdk.Connection) []*cmv1.MachinePool func MachinePoolId(machinePool *cmv1.MachinePool) string { return machinePool.ID() } - -func CheckNodePools(connection *ocmsdk.Connection, awName string) (foundNodePool bool, err error) { - nodePoolsConnection := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).NodePools().List() - nodePoolsListResponse, err := nodePoolsConnection.SendContext(context.Background()) - if err != nil { - return false, fmt.Errorf("unable to send request, error: %v", err) - } - nodePoolsList := nodePoolsListResponse.Items() - nodePoolsList.Range(func(index int, item *cmv1.NodePool) bool { - instanceName, _ := item.GetID() - if strings.Contains(instanceName, awName) { - foundNodePool = true - } - return true - }) - - return foundNodePool, err -} diff --git a/test/support/codeflare.go b/test/support/codeflare.go index c378c4e6..3772ce44 100644 --- a/test/support/codeflare.go +++ b/test/support/codeflare.go @@ -36,6 +36,8 @@ const ( InstaScaleOcmSecretName = "INSTASCALE_OCM_SECRET_NAME" // The namespace where a secret containing InstaScale OCM token is stored. InstaScaleOcmSecretNamespace = "INSTASCALE_OCM_SECRET_NAMESPACE" + // Cluster ID for OSD cluster used in tests, used for testing InstaScale + OsdClusterID = "CLUSTERID" ) func GetCodeFlareSDKVersion() string { @@ -62,6 +64,10 @@ func GetInstaScaleOcmSecretNamespace() string { return lookupEnvOrDefault(InstaScaleOcmSecretNamespace, "default") } +func GetOsdClusterId() (string, bool) { + return os.LookupEnv(OsdClusterID) +} + func lookupEnvOrDefault(key, value string) string { if v, ok := os.LookupEnv(key); ok { return v diff --git a/test/support/config_map.go b/test/support/config_map.go index 8846bec2..392da42b 100644 --- a/test/support/config_map.go +++ b/test/support/config_map.go @@ -18,6 +18,7 @@ package support import ( "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/test/support/ocm.go b/test/support/ocm.go index 235143ba..ff4d0da4 100644 --- a/test/support/ocm.go +++ b/test/support/ocm.go @@ -21,8 +21,10 @@ import ( "os" "github.com/onsi/gomega" - ocmsdk "github.com/openshift-online/ocm-sdk-go" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + ocmsdk "github.com/openshift-online/ocm-sdk-go" ) func CreateOCMConnection(test Test) *ocmsdk.Connection { From 0aa7a73a9d5d7bd93e3f64f8e212aa80afe36908 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Wed, 4 Oct 2023 16:45:55 +0100 Subject: [PATCH 169/377] addressing feedback --- .github/workflows/e2e_tests.yaml | 1 + test/support/ocm.go | 2 +- test/support/support.go | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 2f119bf6..b66035a9 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -65,6 +65,7 @@ jobs: export CODEFLARE_TEST_TIMEOUT_SHORT=1m export CODEFLARE_TEST_TIMEOUT_MEDIUM=5m export CODEFLARE_TEST_TIMEOUT_LONG=10m + export CODEFLARE_TEST_TIMEOUT_GPU_PROVISIONING=30m export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV diff --git a/test/support/ocm.go b/test/support/ocm.go index ff4d0da4..91738fbc 100644 --- a/test/support/ocm.go +++ b/test/support/ocm.go @@ -43,7 +43,7 @@ func buildOCMConnection(secret string) (*ocmsdk.Connection, error) { connection, err := ocmsdk.NewConnectionBuilder(). Tokens(secret). Build() - if err != nil || connection == nil { + if err != nil { fmt.Fprintf(os.Stderr, "Can't build connection: %v\n", err) return nil, err } diff --git a/test/support/support.go b/test/support/support.go index 0091b3bf..36c8c9be 100644 --- a/test/support/support.go +++ b/test/support/support.go @@ -58,6 +58,13 @@ func init() { fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_LONG. Using default value: %s", TestTimeoutLong) } } + if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_GPU_PROVISIONING"); ok { + if duration, err := time.ParseDuration(value); err == nil { + TestTimeoutGpuProvisioning = duration + } else { + fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_GPU_PROVISIONING. Using default value: %s", TestTimeoutGpuProvisioning) + } + } // Gomega settings gomega.SetDefaultEventuallyTimeout(TestTimeoutShort) From c66f1d7197b10225a515eb7feec061dd6f189941 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 5 Oct 2023 12:16:29 +0200 Subject: [PATCH 170/377] Add missing licence headers to InstaScale test files --- test/e2e/instascale_app_wrapper.go | 16 ++++++++++++++++ test/e2e/instascale_machinepool_test.go | 16 ++++++++++++++++ test/support/clusterpools.go | 16 ++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go index d5f14efe..7586d659 100644 --- a/test/e2e/instascale_app_wrapper.go +++ b/test/e2e/instascale_app_wrapper.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index ad4143cf..12349d9d 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( diff --git a/test/support/clusterpools.go b/test/support/clusterpools.go index 9b1f7848..b321aeb0 100644 --- a/test/support/clusterpools.go +++ b/test/support/clusterpools.go @@ -1,3 +1,19 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package support import ( From 7d4eea31d7ccc3e8eb05a710f304cfaa0d887bd6 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 5 Oct 2023 12:30:46 +0100 Subject: [PATCH 171/377] skip test if not OSD --- test/e2e/instascale_machinepool_test.go | 4 ++++ test/support/codeflare.go | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 12349d9d..1f62de0d 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -33,6 +33,10 @@ func TestInstascaleMachinePool(t *testing.T) { test := With(t) test.T().Parallel() + if !IsOsd(test) { + test.T().Skip("Skipping test as not running on an OSD cluster") + } + namespace := test.NewTestNamespace() // Test configuration diff --git a/test/support/codeflare.go b/test/support/codeflare.go index 3772ce44..bf336775 100644 --- a/test/support/codeflare.go +++ b/test/support/codeflare.go @@ -18,6 +18,9 @@ package support import ( "os" + "strconv" + + "github.com/onsi/gomega" ) const ( @@ -38,6 +41,8 @@ const ( InstaScaleOcmSecretNamespace = "INSTASCALE_OCM_SECRET_NAMESPACE" // Cluster ID for OSD cluster used in tests, used for testing InstaScale OsdClusterID = "CLUSTERID" + // Determine if test is being run on an OSD cluster, used for testing InstaScale. + IsOSD = "IS_OSD" ) func GetCodeFlareSDKVersion() string { @@ -68,6 +73,18 @@ func GetOsdClusterId() (string, bool) { return os.LookupEnv(OsdClusterID) } +func IsOsd(test Test) bool { + test.T().Helper() + env := lookupEnvOrDefault(IsOSD, "false") + osd, err := strconv.ParseBool(env) + if err != nil { + test.T().Logf("error parsing IS_OSD environment variable, using default 'false' value, error: %v ", err) + return false + } + test.Expect(err).NotTo(gomega.HaveOccurred()) + return osd +} + func lookupEnvOrDefault(key, value string) string { if v, ok := os.LookupEnv(key); ok { return v From 8c60290676446afb0d5c59752bfb7d176438b0c3 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 6 Oct 2023 15:26:54 +0100 Subject: [PATCH 172/377] addressing feedback --- test/e2e/instascale_app_wrapper.go | 6 --- test/e2e/instascale_machinepool_test.go | 31 ++++--------- test/support/clusterpools.go | 44 ------------------ test/support/config_map.go | 45 ------------------- test/support/core.go | 21 +++++++++ test/support/{codeflare.go => environment.go} | 36 +++++---------- test/support/ocm.go | 24 +++++++++- 7 files changed, 65 insertions(+), 142 deletions(-) delete mode 100644 test/support/clusterpools.go delete mode 100644 test/support/config_map.go rename test/support/{codeflare.go => environment.go} (67%) diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go index 7586d659..f9ec3bd0 100644 --- a/test/e2e/instascale_app_wrapper.go +++ b/test/e2e/instascale_app_wrapper.go @@ -17,7 +17,6 @@ limitations under the License. package e2e import ( - . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" batchv1 "k8s.io/api/batch/v1" @@ -138,11 +137,6 @@ func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, confi } _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) - - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) return job, aw, err } diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 1f62de0d..b6ad0782 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -22,35 +22,29 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - . "github.com/project-codeflare/codeflare-operator/test/support" ) func TestInstascaleMachinePool(t *testing.T) { - test := With(t) test.T().Parallel() - if !IsOsd(test) { + if !IsOsd() { test.T().Skip("Skipping test as not running on an OSD cluster") } namespace := test.NewTestNamespace() // Test configuration - testConfigData := map[string][]byte{ + cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ // pip requirements "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), // MNIST training script "mnist.py": ReadFile(test, "mnist.py"), - } - cm := CreateConfigMap(test, namespace.Name, testConfigData) + }) //create OCM connection connection := CreateOCMConnection(test) - defer connection.Close() // check existing cluster machine pool resources @@ -59,25 +53,18 @@ func TestInstascaleMachinePool(t *testing.T) { ShouldNot(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) // Setup batch job and AppWrapper - job, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) + _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) + + // assert that AppWrapper goes to "Running" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) // look for machine pool with aw name - expect to find it test.Eventually(MachinePools(test, connection), TestTimeoutLong). Should(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) - // Assert that the job has completed - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( - Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), - )) - - // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) diff --git a/test/support/clusterpools.go b/test/support/clusterpools.go deleted file mode 100644 index b321aeb0..00000000 --- a/test/support/clusterpools.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - - ocmsdk "github.com/openshift-online/ocm-sdk-go" - cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" -) - -func MachinePools(t Test, connection *ocmsdk.Connection) func(g gomega.Gomega) []*cmv1.MachinePool { - osdClusterId, found := GetOsdClusterId() - t.Expect(found).To(gomega.BeTrue(), "OSD cluster id not found, please configure environment properly") - - return func(g gomega.Gomega) []*cmv1.MachinePool { - machinePoolsListResponse, err := connection.ClustersMgmt().V1().Clusters().Cluster(osdClusterId).MachinePools().List().Send() - g.Expect(err).NotTo(gomega.HaveOccurred()) - return machinePoolsListResponse.Items().Slice() - } -} - -func GetMachinePools(t Test, connection *ocmsdk.Connection) []*cmv1.MachinePool { - t.T().Helper() - return MachinePools(t, connection)(t) -} - -func MachinePoolId(machinePool *cmv1.MachinePool) string { - return machinePool.ID() -} diff --git a/test/support/config_map.go b/test/support/config_map.go deleted file mode 100644 index 392da42b..00000000 --- a/test/support/config_map.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func CreateConfigMap(t Test, namespace string, content map[string][]byte) *corev1.ConfigMap { - configMap := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "config-", - Namespace: namespace, - }, - BinaryData: content, - Immutable: Ptr(true), - } - - configMap, err := t.Client().Core().CoreV1().ConfigMaps(namespace).Create(t.Ctx(), configMap, metav1.CreateOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - t.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) - - return configMap -} diff --git a/test/support/core.go b/test/support/core.go index ee012c82..70c48c20 100644 --- a/test/support/core.go +++ b/test/support/core.go @@ -27,6 +27,27 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +func CreateConfigMap(t Test, namespace string, content map[string][]byte) *corev1.ConfigMap { + configMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "config-", + Namespace: namespace, + }, + BinaryData: content, + Immutable: Ptr(true), + } + + configMap, err := t.Client().Core().CoreV1().ConfigMaps(namespace).Create(t.Ctx(), configMap, metav1.CreateOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + t.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) + + return configMap +} + func Raw(t Test, obj runtime.Object) runtime.RawExtension { t.T().Helper() data, err := json.Marshal(obj) diff --git a/test/support/codeflare.go b/test/support/environment.go similarity index 67% rename from test/support/codeflare.go rename to test/support/environment.go index bf336775..bf3b2af7 100644 --- a/test/support/codeflare.go +++ b/test/support/environment.go @@ -18,9 +18,7 @@ package support import ( "os" - "strconv" - - "github.com/onsi/gomega" + "strings" ) const ( @@ -35,14 +33,11 @@ const ( // The testing output directory, to write output files into. CodeFlareTestOutputDir = "CODEFLARE_TEST_OUTPUT_DIR" - // The name of a secret containing InstaScale OCM token. - InstaScaleOcmSecretName = "INSTASCALE_OCM_SECRET_NAME" - // The namespace where a secret containing InstaScale OCM token is stored. - InstaScaleOcmSecretNamespace = "INSTASCALE_OCM_SECRET_NAMESPACE" + // The namespace where a secret containing InstaScale OCM token is stored and the secret name. + InstaScaleOcmSecret = "INSTASCALE_OCM_SECRET" + // Cluster ID for OSD cluster used in tests, used for testing InstaScale OsdClusterID = "CLUSTERID" - // Determine if test is being run on an OSD cluster, used for testing InstaScale. - IsOSD = "IS_OSD" ) func GetCodeFlareSDKVersion() string { @@ -61,28 +56,21 @@ func GetPyTorchImage() string { return lookupEnvOrDefault(CodeFlareTestPyTorchImage, "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime") } -func GetInstaScaleOcmSecretName() string { - return lookupEnvOrDefault(InstaScaleOcmSecretName, "instascale-ocm-secret") -} - -func GetInstaScaleOcmSecretNamespace() string { - return lookupEnvOrDefault(InstaScaleOcmSecretNamespace, "default") +func GetInstascaleOcmSecret() (string, string) { + res := strings.SplitN(lookupEnvOrDefault(InstaScaleOcmSecret, "default/instascale-com-secret"), "/", 2) + return res[0], res[1] } func GetOsdClusterId() (string, bool) { return os.LookupEnv(OsdClusterID) } -func IsOsd(test Test) bool { - test.T().Helper() - env := lookupEnvOrDefault(IsOSD, "false") - osd, err := strconv.ParseBool(env) - if err != nil { - test.T().Logf("error parsing IS_OSD environment variable, using default 'false' value, error: %v ", err) - return false +func IsOsd() bool { + osdClusterId, found := GetOsdClusterId() + if found && osdClusterId != "" { + return true } - test.Expect(err).NotTo(gomega.HaveOccurred()) - return osd + return false } func lookupEnvOrDefault(key, value string) string { diff --git a/test/support/ocm.go b/test/support/ocm.go index 91738fbc..3d6bd7f1 100644 --- a/test/support/ocm.go +++ b/test/support/ocm.go @@ -25,10 +25,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ocmsdk "github.com/openshift-online/ocm-sdk-go" + cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" ) func CreateOCMConnection(test Test) *ocmsdk.Connection { - instascaleOCMSecret, err := test.Client().Core().CoreV1().Secrets(GetInstaScaleOcmSecretNamespace()).Get(test.Ctx(), GetInstaScaleOcmSecretName(), metav1.GetOptions{}) + secretNamespace, secretName := GetInstascaleOcmSecret() + instascaleOCMSecret, err := test.Client().Core().CoreV1().Secrets(secretNamespace).Get(test.Ctx(), secretName, metav1.GetOptions{}) test.Expect(err).NotTo(gomega.HaveOccurred()) ocmToken := string(instascaleOCMSecret.Data["token"]) @@ -50,3 +52,23 @@ func buildOCMConnection(secret string) (*ocmsdk.Connection, error) { return connection, nil } + +func MachinePools(t Test, connection *ocmsdk.Connection) func(g gomega.Gomega) []*cmv1.MachinePool { + osdClusterId, found := GetOsdClusterId() + t.Expect(found).To(gomega.BeTrue(), "OSD cluster id not found, please configure environment properly") + + return func(g gomega.Gomega) []*cmv1.MachinePool { + machinePoolsListResponse, err := connection.ClustersMgmt().V1().Clusters().Cluster(osdClusterId).MachinePools().List().Send() + g.Expect(err).NotTo(gomega.HaveOccurred()) + return machinePoolsListResponse.Items().Slice() + } +} + +func GetMachinePools(t Test, connection *ocmsdk.Connection) []*cmv1.MachinePool { + t.T().Helper() + return MachinePools(t, connection)(t) +} + +func MachinePoolId(machinePool *cmv1.MachinePool) string { + return machinePool.ID() +} From 2ddc14f16c9ab7a5553b5fb8af922430e94f234c Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 9 Oct 2023 11:10:49 +0100 Subject: [PATCH 173/377] fix typo in ocm secret name --- test/support/environment.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/support/environment.go b/test/support/environment.go index bf3b2af7..54074982 100644 --- a/test/support/environment.go +++ b/test/support/environment.go @@ -57,7 +57,7 @@ func GetPyTorchImage() string { } func GetInstascaleOcmSecret() (string, string) { - res := strings.SplitN(lookupEnvOrDefault(InstaScaleOcmSecret, "default/instascale-com-secret"), "/", 2) + res := strings.SplitN(lookupEnvOrDefault(InstaScaleOcmSecret, "default/instascale-ocm-secret"), "/", 2) return res[0], res[1] } From 25b9606955a1a614967d46423cde32334c26b384 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 25 Sep 2023 12:24:53 +0200 Subject: [PATCH 174/377] Update compute resources to account for MCAD and InstaScale --- config/manager/manager.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ca1823c3..ae13f8af 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -65,10 +65,10 @@ spec: periodSeconds: 10 resources: limits: - cpu: 500m - memory: 128Mi + cpu: "1" + memory: 1Gi requests: - cpu: 10m - memory: 64Mi + cpu: "1" + memory: 1Gi serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 From b1e27deb11e9d4df859600e72b833c520b85cb1d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 25 Sep 2023 15:38:52 +0200 Subject: [PATCH 175/377] Patch Deployment resources for e2e tests --- .github/workflows/e2e_tests.yaml | 2 +- Makefile | 8 ++++++-- config/e2e/kustomization.yaml | 9 +++++++++ config/e2e/patch_resources.yaml | 2 ++ 4 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 config/e2e/kustomization.yaml create mode 100644 config/e2e/patch_resources.yaml diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index b66035a9..64d4567e 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -54,7 +54,7 @@ jobs: echo Deploying CodeFlare operator IMG="${REGISTRY_ADDRESS}"/codeflare-operator make image-push -e IMG="${IMG}" - make deploy -e IMG="${IMG}" + make deploy -e IMG="${IMG}" -e ENV="e2e" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager echo Setting up CodeFlare stack diff --git a/Makefile b/Makefile index c7b10256..df5842e6 100644 --- a/Makefile +++ b/Makefile @@ -87,6 +87,10 @@ IMG ?= ${IMAGE_TAG_BASE}:${VERSION} # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.24.2 +# The target deployment environment, that corresponds to the Kustomize directory +# used to build the manifests. +ENV ?= default + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -202,13 +206,13 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | kubectl apply -f - + $(KUSTOMIZE) build config/${ENV} | kubectl apply -f - git restore config/* .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml - $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + $(KUSTOMIZE) build config/${ENV} | kubectl delete --ignore-not-found=$(ignore-not-found) -f - git restore config/* ##@ Build Dependencies diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml new file mode 100644 index 00000000..faf613e7 --- /dev/null +++ b/config/e2e/kustomization.yaml @@ -0,0 +1,9 @@ +bases: +- ../default + +patches: + - target: + kind: Deployment + name: manager + namespace: system + path: patch_resources.yaml diff --git a/config/e2e/patch_resources.yaml b/config/e2e/patch_resources.yaml new file mode 100644 index 00000000..3da1d5f3 --- /dev/null +++ b/config/e2e/patch_resources.yaml @@ -0,0 +1,2 @@ +- op: remove + path: /spec/template/spec/containers/0/resources From 2b6f4947d1cfedaf6d64336146ad2fe529267d76 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 25 Sep 2023 15:44:05 +0200 Subject: [PATCH 176/377] Overwrite Subscription resources in OLM e2e test --- .github/resources-olm-upgrade/subscription.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/resources-olm-upgrade/subscription.yaml b/.github/resources-olm-upgrade/subscription.yaml index 1caf8794..07fbb23b 100644 --- a/.github/resources-olm-upgrade/subscription.yaml +++ b/.github/resources-olm-upgrade/subscription.yaml @@ -9,3 +9,11 @@ spec: name: codeflare-operator source: codeflare-olm-test sourceNamespace: olm + config: + resources: + limits: + cpu: 400m + memory: 128Mi + requests: + cpu: 50m + memory: 64Mi From 2ed79fb8ca3d67175808e809f1bfb7efec1023c8 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 27 Sep 2023 17:51:39 +0200 Subject: [PATCH 177/377] e2e: Upgrade OLM to version v0.25.0 --- .github/workflows/olm_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 5002f069..382c95f3 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-20.04-4core timeout-minutes: 60 env: - OLM_VERSION: v0.24.0 + OLM_VERSION: v0.25.0 VERSION: "v0.0.0-ghaction" # Need to supply some semver version for bundle to be properly generated CATALOG_BASE_IMG: "registry.access.redhat.com/redhat/community-operator-index:v4.13" CODEFLARE_TEST_TIMEOUT_SHORT: "1m" From bd0c2b229e68376df8c3f495cda11c76133c5aac Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 11 Oct 2023 15:34:55 -0400 Subject: [PATCH 178/377] remove arg for quay org, IS and MCAD images no longer being built Signed-off-by: Kevin --- .github/workflows/project-codeflare-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 286255d1..dd94ebcf 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -51,7 +51,7 @@ jobs: - name: Release MCAD run: | - gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref ${{ github.ref }} --field tag=${{ github.event.inputs.mcad-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} + gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref ${{ github.ref }} --field tag=${{ github.event.inputs.mcad-version }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash @@ -83,7 +83,7 @@ jobs: - name: Release InstaScale run: | - gh workflow run instascale-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field tag=${{ github.event.inputs.instascale-version }} --field mcad-version=${{ github.event.inputs.mcad-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} + gh workflow run instascale-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --ref ${{ github.ref }} --field tag=${{ github.event.inputs.instascale-version }} --field mcad-version=${{ github.event.inputs.mcad-version }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash From 04d4b430b6b5d277d4a5debd92546c7a9973beaa Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Thu, 12 Oct 2023 12:21:52 +0100 Subject: [PATCH 179/377] add: Print out parameters used in CodeFlare release workflow --- .github/workflows/project-codeflare-release.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index dd94ebcf..1f8bb0c7 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -37,6 +37,22 @@ on: default: 'redhat-openshift-ecosystem' jobs: + release-parameters: + runs-on: ubuntu-latest + steps: + - name: Release Parameters + run: | + echo "Below are the release parameters set for the workflow:" + echo "Operator Version: ${{ github.event.inputs.operator-version }}" + echo "Replaces: ${{ github.event.inputs.replaces }}" + echo "MCAD Version: ${{ github.event.inputs.mcad-version }}" + echo "CodeFlare SDK Version: ${{ github.event.inputs.codeflare-sdk-version }}" + echo "InstaScale Version: ${{ github.event.inputs.instascale-version }}" + echo "Is Stable: ${{ github.event.inputs.is-stable }}" + echo "CodeFlare Repository Organization: ${{ github.event.inputs.codeflare-repository-organization }}" + echo "Quay Organization: ${{ github.event.inputs.quay-organization }}" + echo "Community Operators Prod Organization: ${{ github.event.inputs.community-operators-prod-organization }}" + release-mcad: runs-on: ubuntu-latest From 607ad9eae96488afe5a52c0a47e00f9fb01b3ab0 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Thu, 12 Oct 2023 13:26:06 +0000 Subject: [PATCH 180/377] Update dependency versions for release v1.0.0-rc.2 --- Makefile | 6 +++--- README.md | 8 ++++---- go.mod | 6 +++--- go.sum | 12 ++++++------ test/support/defaults.go | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index df5842e6..725b661a 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.0.9 +INSTASCALE_VERSION ?= v0.1.0 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= 725a614debe3d34d1547c1659ef5ad49f8f6c5df +MCAD_VERSION ?= v1.36.0 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} @@ -28,7 +28,7 @@ KUBERAY_VERSION ?= v0.6.0 RAY_VERSION ?= 2.5.0 # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK -CODEFLARE_SDK_VERSION ?= 0.8.0 +CODEFLARE_SDK_VERSION ?= 0.9.0 # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to diff --git a/README.md b/README.md index 75ed7d4f..b2f40931 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.0.0-rc.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.1) | -| Multi-Cluster App Dispatcher | [v1.35.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.35.0) | -| CodeFlare-SDK | [v0.8.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.8.0) | -| InstaScale | [v0.0.9](https://github.com/project-codeflare/instascale/releases/tag/v0.0.9) | +| CodeFlare Operator | [v1.0.0-rc.2](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.2) | +| Multi-Cluster App Dispatcher | [v1.36.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.36.0) | +| CodeFlare-SDK | [v0.9.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.9.0) | +| InstaScale | [v0.1.0](https://github.com/project-codeflare/instascale/releases/tag/v0.1.0) | | KubeRay | [v0.5.0](https://github.com/ray-project/kuberay/releases/tag/v0.5.0) | diff --git a/go.mod b/go.mod index 01d9e1b9..9166e25a 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,9 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.368 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/instascale v0.0.9 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.1-0.20231004084320-725a614debe3 - github.com/ray-project/kuberay/ray-operator v0.0.0-20231003160009-38e3527cfce7 + github.com/project-codeflare/instascale v0.1.0 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.36.0 + github.com/ray-project/kuberay/ray-operator v0.0.0-20231012011224-15ce568a0c36 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index e564c6f8..f5d91a72 100644 --- a/go.sum +++ b/go.sum @@ -389,10 +389,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/instascale v0.0.9 h1:XlwprvuJNbNVnLxCa6lPz5bKUEMhJprVRumd42sRhVY= -github.com/project-codeflare/instascale v0.0.9/go.mod h1:fpc8TiFvR2r0duKFbKmGTshEGa0eGK9noWWfjhPxphE= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.1-0.20231004084320-725a614debe3 h1:l/QHuyrSgaZfZ1UCRjj49rhagJV7tsyNlq0+lRhS6ZM= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.35.1-0.20231004084320-725a614debe3/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/instascale v0.1.0 h1:oP5PwUIDvh+D3fbg8sT9dKphl9K/Tb/RraTFQezLOxI= +github.com/project-codeflare/instascale v0.1.0/go.mod h1:c4I8w4R0I8OJdkwA/ht/o/3lxRCBx4YdFJ3D1DulFVA= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.36.0 h1:QSc16Kd4OBidGY4rQivMrUX9B9cNKttfZIRKsR4BX9E= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.36.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -420,8 +420,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/ray-project/kuberay/ray-operator v0.0.0-20231003160009-38e3527cfce7 h1:arS5d3gMlpVJTcBipREs6uOWP4mL7C0EtQutISpC4Mo= -github.com/ray-project/kuberay/ray-operator v0.0.0-20231003160009-38e3527cfce7/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= +github.com/ray-project/kuberay/ray-operator v0.0.0-20231012011224-15ce568a0c36 h1:IzXHzI4OjL1ySpvmPHTWjFjmgBBjAZXBa3cQwejLPWY= +github.com/ray-project/kuberay/ray-operator v0.0.0-20231012011224-15ce568a0c36/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= diff --git a/test/support/defaults.go b/test/support/defaults.go index c93a3f48..1beb7e89 100644 --- a/test/support/defaults.go +++ b/test/support/defaults.go @@ -5,7 +5,7 @@ package support // *********************** const ( - CodeFlareSDKVersion = "0.8.0" + CodeFlareSDKVersion = "0.9.0" RayVersion = "2.5.0" RayImage = "rayproject/ray:2.5.0" ) From 79221d8daef789d3982cb7c5bdf8142ef7a0ac35 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 12 Oct 2023 15:26:34 -0400 Subject: [PATCH 181/377] Update go-toolset version for CVEs Signed-off-by: Anish Asthana --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b949b1bd..df237662 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.access.redhat.com/ubi8/go-toolset:1.19.10-10 as builder +FROM registry.access.redhat.com/ubi8/go-toolset:1.19.10-16 as builder WORKDIR /workspace # Copy the Go Modules manifests @@ -15,7 +15,7 @@ COPY pkg/ pkg/ USER root RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7 +FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8 WORKDIR / COPY --from=builder /workspace/manager . From 7c5efc022a43210020f534cb6d38d3c3bfd86c22 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 13 Oct 2023 08:49:41 +0200 Subject: [PATCH 182/377] Adjust the SDK PR link as a reason to skip SDK e2e test --- test/e2e/mnist_raycluster_sdk_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 8a3bc390..e9611065 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -41,8 +41,8 @@ func TestMNISTRayClusterSDK(t *testing.T) { test := With(t) test.T().Parallel() - // Currently blocked by https://github.com/project-codeflare/codeflare-sdk/pull/271 , remove the skip once SDK with the PR is released - test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/271") + // Currently blocked by https://github.com/project-codeflare/codeflare-sdk/pull/251 , remove the skip once SDK with the PR is released + test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/251") // Create a namespace namespace := test.NewTestNamespace() From b6c158237ddfd295fd7f6b57bfa1641e579ae63d Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 16 Oct 2023 15:46:43 +0100 Subject: [PATCH 183/377] add ClusterRole and ClusterRoleBinding for KubeRay --- config/rbac/kustomization.yaml | 2 ++ .../rbac/mcad-controller-ray-clusterrole.yaml | 31 +++++++++++++++++++ ...cad-controller-ray-clusterrolebinding.yaml | 12 +++++++ 3 files changed, 45 insertions(+) create mode 100644 config/rbac/mcad-controller-ray-clusterrole.yaml create mode 100644 config/rbac/mcad-controller-ray-clusterrolebinding.yaml diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 05bf4c6c..cdb307c8 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -11,3 +11,5 @@ resources: - instascale_role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml +- mcad-controller-ray-clusterrolebinding.yaml +- mcad-controller-ray-clusterrole.yaml diff --git a/config/rbac/mcad-controller-ray-clusterrole.yaml b/config/rbac/mcad-controller-ray-clusterrole.yaml new file mode 100644 index 00000000..f9562420 --- /dev/null +++ b/config/rbac/mcad-controller-ray-clusterrole.yaml @@ -0,0 +1,31 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mcad-controller-ray-clusterrole +rules: +- apiGroups: + - ray.io + resources: + - rayclusters + - rayclusters/finalizers + - rayclusters/status + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml new file mode 100644 index 00000000..a3931da0 --- /dev/null +++ b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: mcad-controller-ray-clusterrolebinding +subjects: + - kind: ServiceAccount + name: codeflare-operator-controller-manager + namespace: openshift-operators +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: mcad-controller-ray-clusterrole From 4f21b11efe291699cea6ef1091b88b6695b4ab57 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 6 Oct 2023 13:44:01 +0100 Subject: [PATCH 184/377] Test Instascale machine set functionality Co-authored-by: Karel Suta ksuta@redhat.com Co-authored-by: Fiona Waters fwaters@redhat.com --- test/e2e/instascale_machineset_test.go | 54 ++++++++++++++++++++++++++ test/support/client.go | 13 +++++++ test/support/machinesets.go | 36 +++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 test/e2e/instascale_machineset_test.go create mode 100644 test/support/machinesets.go diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go new file mode 100644 index 00000000..08805555 --- /dev/null +++ b/test/e2e/instascale_machineset_test.go @@ -0,0 +1,54 @@ +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + + . "github.com/project-codeflare/codeflare-operator/test/support" +) + +func TestInstascaleMachineSet(t *testing.T) { + test := With(t) + test.T().Parallel() + + // skip test if not using machine sets + if !MachineSetsExist(test) { + test.T().Skip("Skipping test as machine sets don't exist") + } + + namespace := test.NewTestNamespace() + + // Test configuration + cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }) + + // look for machine set with aw name - expect to find it + test.Expect(MachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale")))) + // look for machine set replica - expect not to find it + test.Expect(MachineExists(test)).Should(BeFalse()) + + // // Setup batch job and AppWrapper + _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) + test.Expect(err).NotTo(HaveOccurred()) + + // assert that AppWrapper goes to "Running" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + //look for machine set replica - expect to find it + test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeTrue()) + + // assert that the AppWrapper goes to "Completed" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) + + // look for machine set replica - expect not to find it + test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeFalse()) + +} diff --git a/test/support/client.go b/test/support/client.go index f5696bde..d911c7c2 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -26,6 +26,7 @@ import ( "k8s.io/client-go/tools/clientcmd" imagev1 "github.com/openshift/client-go/image/clientset/versioned" + machinev1 "github.com/openshift/client-go/machine/clientset/versioned" routev1 "github.com/openshift/client-go/route/clientset/versioned" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -33,6 +34,7 @@ import ( type Client interface { Core() kubernetes.Interface + Machine() machinev1.Interface Route() routev1.Interface Image() imagev1.Interface MCAD() mcadclient.Interface @@ -42,6 +44,7 @@ type Client interface { type testClient struct { core kubernetes.Interface + machine machinev1.Interface route routev1.Interface image imagev1.Interface mcad mcadclient.Interface @@ -55,6 +58,10 @@ func (t *testClient) Core() kubernetes.Interface { return t.core } +func (t *testClient) Machine() machinev1.Interface { + return t.machine +} + func (t *testClient) Route() routev1.Interface { return t.route } @@ -88,6 +95,11 @@ func newTestClient() (Client, error) { return nil, err } + machineClient, err := machinev1.NewForConfig(cfg) + if err != nil { + return nil, err + } + routeClient, err := routev1.NewForConfig(cfg) if err != nil { return nil, err @@ -115,6 +127,7 @@ func newTestClient() (Client, error) { return &testClient{ core: kubeClient, + machine: machineClient, route: routeClient, image: imageClient, mcad: mcadClient, diff --git a/test/support/machinesets.go b/test/support/machinesets.go new file mode 100644 index 00000000..b4545536 --- /dev/null +++ b/test/support/machinesets.go @@ -0,0 +1,36 @@ +package support + +import ( + "github.com/onsi/gomega" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1beta1 "github.com/openshift/api/machine/v1beta1" +) + +func MachineSets(t Test) ([]v1beta1.MachineSet, error) { + ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), v1.ListOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms.Items, err +} + +func MachineExists(t Test) (foundReplica bool) { + machineSetList, err := MachineSets(t) + t.Expect(err).NotTo(gomega.HaveOccurred()) + for _, ms := range machineSetList { + if ms.Name == "test-instascale" && &ms.Status.AvailableReplicas == Ptr(int32(1)) { + foundReplica = true + } + } + return foundReplica +} + +func MachineSetId(machineSet v1beta1.MachineSet) string { + return machineSet.Name +} + +func MachineSetsExist(t Test) bool { + ms, err := MachineSets(t) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms != nil +} From 55be710877db3862f29bd11e0d64f93a0311fa9e Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Wed, 11 Oct 2023 10:17:53 +0100 Subject: [PATCH 185/377] catching 404 error when machine sets not in use --- test/e2e/instascale_machineset_test.go | 5 ++++- test/support/machinesets.go | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index 08805555..0a1975a4 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -6,6 +6,8 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -14,7 +16,8 @@ func TestInstascaleMachineSet(t *testing.T) { test.T().Parallel() // skip test if not using machine sets - if !MachineSetsExist(test) { + ms, err := MachineSetsExist(test) + if !ms || err != nil && errors.IsNotFound(err) { test.T().Skip("Skipping test as machine sets don't exist") } diff --git a/test/support/machinesets.go b/test/support/machinesets.go index b4545536..85259536 100644 --- a/test/support/machinesets.go +++ b/test/support/machinesets.go @@ -3,6 +3,7 @@ package support import ( "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1beta1 "github.com/openshift/api/machine/v1beta1" @@ -29,8 +30,11 @@ func MachineSetId(machineSet v1beta1.MachineSet) string { return machineSet.Name } -func MachineSetsExist(t Test) bool { +func MachineSetsExist(t Test) (bool, error) { ms, err := MachineSets(t) + if err != nil && errors.IsNotFound(err) { + return false, err + } t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms != nil + return ms != nil, err } From 4d1f8ec8d44b5dd81c79d910823b61986f637334 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 11 Oct 2023 17:52:30 +0200 Subject: [PATCH 186/377] Use generic function to retrieve machines --- test/e2e/instascale_machineset_test.go | 16 +++++----- test/support/machine.go | 42 ++++++++++++++++++++++++++ test/support/machinesets.go | 40 ------------------------ 3 files changed, 50 insertions(+), 48 deletions(-) create mode 100644 test/support/machine.go delete mode 100644 test/support/machinesets.go diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index 0a1975a4..df79aec2 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -32,9 +32,9 @@ func TestInstascaleMachineSet(t *testing.T) { }) // look for machine set with aw name - expect to find it - test.Expect(MachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale")))) - // look for machine set replica - expect not to find it - test.Expect(MachineExists(test)).Should(BeFalse()) + test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale")))) + // look for machine belonging to the machine set, there should be none + test.Expect(GetMachines(test, "test-instascale")).Should(BeEmpty()) // // Setup batch job and AppWrapper _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) @@ -44,14 +44,14 @@ func TestInstascaleMachineSet(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - //look for machine set replica - expect to find it - test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeTrue()) + // look for machine belonging to the machine set - expect to find it + test.Eventually(Machines(test, "test-instascale"), TestTimeoutLong).Should(HaveLen(1)) // assert that the AppWrapper goes to "Completed" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - // look for machine set replica - expect not to find it - test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeFalse()) + // look for machine belonging to the machine set - there should be none + test.Eventually(Machines(test, "test-instascale"), TestTimeoutLong).Should(BeEmpty()) } diff --git a/test/support/machine.go b/test/support/machine.go new file mode 100644 index 00000000..f6be3750 --- /dev/null +++ b/test/support/machine.go @@ -0,0 +1,42 @@ +package support + +import ( + "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + machinev1beta1 "github.com/openshift/api/machine/v1beta1" +) + +func GetMachineSets(t Test) ([]machinev1beta1.MachineSet, error) { + ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms.Items, err +} + +func Machines(t Test, machineSetName string) func(g gomega.Gomega) []machinev1beta1.Machine { + return func(g gomega.Gomega) []machinev1beta1.Machine { + machine, err := t.Client().Machine().MachineV1beta1().Machines("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{LabelSelector: "machine.openshift.io/cluster-api-machineset=" + machineSetName}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return machine.Items + } +} + +func GetMachines(t Test, machineSetName string) []machinev1beta1.Machine { + t.T().Helper() + return Machines(t, machineSetName)(t) +} + +func MachineSetId(machineSet machinev1beta1.MachineSet) string { + return machineSet.Name +} + +func MachineSetsExist(t Test) (bool, error) { + ms, err := GetMachineSets(t) + if err != nil && errors.IsNotFound(err) { + return false, err + } + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms != nil, err +} diff --git a/test/support/machinesets.go b/test/support/machinesets.go deleted file mode 100644 index 85259536..00000000 --- a/test/support/machinesets.go +++ /dev/null @@ -1,40 +0,0 @@ -package support - -import ( - "github.com/onsi/gomega" - - "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - v1beta1 "github.com/openshift/api/machine/v1beta1" -) - -func MachineSets(t Test) ([]v1beta1.MachineSet, error) { - ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), v1.ListOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms.Items, err -} - -func MachineExists(t Test) (foundReplica bool) { - machineSetList, err := MachineSets(t) - t.Expect(err).NotTo(gomega.HaveOccurred()) - for _, ms := range machineSetList { - if ms.Name == "test-instascale" && &ms.Status.AvailableReplicas == Ptr(int32(1)) { - foundReplica = true - } - } - return foundReplica -} - -func MachineSetId(machineSet v1beta1.MachineSet) string { - return machineSet.Name -} - -func MachineSetsExist(t Test) (bool, error) { - ms, err := MachineSets(t) - if err != nil && errors.IsNotFound(err) { - return false, err - } - t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms != nil, err -} From 24a400f82e28210e4dbdba3a3490f5947f9415b0 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 12 Oct 2023 11:41:42 +0100 Subject: [PATCH 187/377] determining cluster type --- test/e2e/instascale_machineset_test.go | 8 +++---- test/support/environment.go | 30 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index df79aec2..aef1dfe7 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -6,8 +6,6 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - "k8s.io/apimachinery/pkg/api/errors" - . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -16,9 +14,9 @@ func TestInstascaleMachineSet(t *testing.T) { test.T().Parallel() // skip test if not using machine sets - ms, err := MachineSetsExist(test) - if !ms || err != nil && errors.IsNotFound(err) { - test.T().Skip("Skipping test as machine sets don't exist") + clusterType := DetermineClusterType() + if clusterType != OcpCluster { + test.T().Skip("Skipping test as not running on an OCP cluster") } namespace := test.NewTestNamespace() diff --git a/test/support/environment.go b/test/support/environment.go index 54074982..2455603a 100644 --- a/test/support/environment.go +++ b/test/support/environment.go @@ -38,6 +38,15 @@ const ( // Cluster ID for OSD cluster used in tests, used for testing InstaScale OsdClusterID = "CLUSTERID" + + // Type of cluster test is run on + ClusterType = "CLUSTER_TYPE" + // OpenShift Dedicated Cluster + OsdCluster = "OSD" + // OpenShift Container Platform Cluster + OcpCluster = "OCP" + // ROSA Hosted Hypershift Cluster + HypershiftCluster = "HYPERSHIFT" ) func GetCodeFlareSDKVersion() string { @@ -65,6 +74,10 @@ func GetOsdClusterId() (string, bool) { return os.LookupEnv(OsdClusterID) } +func GetClusterType() (string, bool) { + return os.LookupEnv(ClusterType) +} + func IsOsd() bool { osdClusterId, found := GetOsdClusterId() if found && osdClusterId != "" { @@ -79,3 +92,20 @@ func lookupEnvOrDefault(key, value string) string { } return value } + +func DetermineClusterType() string { + clusterType, ok := GetClusterType() + if !ok { + return "" + } + switch clusterType { + case "OSD": + return OsdCluster + case "OCP": + return OcpCluster + case "HYPERSHIFT": + return HypershiftCluster + default: + return "" + } +} From b5f8b04fc864758af8be55aea255d80fc977170d Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 12 Oct 2023 16:00:50 +0200 Subject: [PATCH 188/377] Use enum for cluster type in e2e tests --- test/e2e/instascale_machineset_test.go | 4 +- test/support/environment.go | 53 +++++++++++++------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index aef1dfe7..8e23d4e7 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -14,9 +14,9 @@ func TestInstascaleMachineSet(t *testing.T) { test.T().Parallel() // skip test if not using machine sets - clusterType := DetermineClusterType() + clusterType := GetClusterType(test) if clusterType != OcpCluster { - test.T().Skip("Skipping test as not running on an OCP cluster") + test.T().Skipf("Skipping test as not running on an OCP cluster, resolved cluster type: %s", clusterType) } namespace := test.NewTestNamespace() diff --git a/test/support/environment.go b/test/support/environment.go index 2455603a..cdafca6e 100644 --- a/test/support/environment.go +++ b/test/support/environment.go @@ -40,13 +40,16 @@ const ( OsdClusterID = "CLUSTERID" // Type of cluster test is run on - ClusterType = "CLUSTER_TYPE" - // OpenShift Dedicated Cluster - OsdCluster = "OSD" - // OpenShift Container Platform Cluster - OcpCluster = "OCP" - // ROSA Hosted Hypershift Cluster - HypershiftCluster = "HYPERSHIFT" + ClusterTypeEnvVar = "CLUSTER_TYPE" +) + +type ClusterType string + +const ( + OsdCluster ClusterType = "OSD" + OcpCluster ClusterType = "OCP" + HypershiftCluster ClusterType = "HYPERSHIFT" + UndefinedCluster ClusterType = "UNDEFINED" ) func GetCodeFlareSDKVersion() string { @@ -74,8 +77,23 @@ func GetOsdClusterId() (string, bool) { return os.LookupEnv(OsdClusterID) } -func GetClusterType() (string, bool) { - return os.LookupEnv(ClusterType) +func GetClusterType(t Test) ClusterType { + clusterType, ok := os.LookupEnv(ClusterTypeEnvVar) + if !ok { + t.T().Logf("Expected environment variable %s not found, cluster type is not defined.", ClusterTypeEnvVar) + return UndefinedCluster + } + switch clusterType { + case "OSD": + return OsdCluster + case "OCP": + return OcpCluster + case "HYPERSHIFT": + return HypershiftCluster + default: + t.T().Logf("Expected environment variable %s contains unexpected value: '%s'", ClusterTypeEnvVar, clusterType) + return UndefinedCluster + } } func IsOsd() bool { @@ -92,20 +110,3 @@ func lookupEnvOrDefault(key, value string) string { } return value } - -func DetermineClusterType() string { - clusterType, ok := GetClusterType() - if !ok { - return "" - } - switch clusterType { - case "OSD": - return OsdCluster - case "OCP": - return OcpCluster - case "HYPERSHIFT": - return HypershiftCluster - default: - return "" - } -} From 90a0bbf16f33e041a400bd49469208bfd595a48f Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 12 Oct 2023 15:20:14 +0100 Subject: [PATCH 189/377] updating machinepool test skip --- test/e2e/instascale_machinepool_test.go | 5 +++-- test/support/environment.go | 8 -------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index b6ad0782..4c2cbab5 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -29,8 +29,9 @@ func TestInstascaleMachinePool(t *testing.T) { test := With(t) test.T().Parallel() - if !IsOsd() { - test.T().Skip("Skipping test as not running on an OSD cluster") + clusterType := GetClusterType(test) + if clusterType != OsdCluster { + test.T().Skipf("Skipping test as not running on an OSD cluster, resolved cluster type: %s", clusterType) } namespace := test.NewTestNamespace() diff --git a/test/support/environment.go b/test/support/environment.go index cdafca6e..7d8e6bc8 100644 --- a/test/support/environment.go +++ b/test/support/environment.go @@ -96,14 +96,6 @@ func GetClusterType(t Test) ClusterType { } } -func IsOsd() bool { - osdClusterId, found := GetOsdClusterId() - if found && osdClusterId != "" { - return true - } - return false -} - func lookupEnvOrDefault(key, value string) string { if v, ok := os.LookupEnv(key); ok { return v From da0cf86e8847037746d06d810e0c5cf153889e08 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 12 Oct 2023 15:35:01 +0100 Subject: [PATCH 190/377] removing unused function --- test/support/machine.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test/support/machine.go b/test/support/machine.go index f6be3750..26acb4bc 100644 --- a/test/support/machine.go +++ b/test/support/machine.go @@ -3,7 +3,6 @@ package support import ( "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" machinev1beta1 "github.com/openshift/api/machine/v1beta1" @@ -31,12 +30,3 @@ func GetMachines(t Test, machineSetName string) []machinev1beta1.Machine { func MachineSetId(machineSet machinev1beta1.MachineSet) string { return machineSet.Name } - -func MachineSetsExist(t Test) (bool, error) { - ms, err := GetMachineSets(t) - if err != nil && errors.IsNotFound(err) { - return false, err - } - t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms != nil, err -} From ea777c6416da985ddab3fcf6467d6d8ca806d850 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 16 Oct 2023 12:53:18 +0100 Subject: [PATCH 191/377] refactor to use aw.Name --- test/e2e/instascale_app_wrapper.go | 8 +++----- test/e2e/instascale_machinepool_test.go | 7 ++++++- test/e2e/instascale_machineset_test.go | 17 +++++++++++------ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go index f9ec3bd0..805df1bc 100644 --- a/test/e2e/instascale_app_wrapper.go +++ b/test/e2e/instascale_app_wrapper.go @@ -27,7 +27,7 @@ import ( . "github.com/project-codeflare/codeflare-operator/test/support" ) -func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) (*batchv1.Job, *mcadv1beta1.AppWrapper, error) { +func instaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) *mcadv1beta1.AppWrapper { // Batch Job job := &batchv1.Job{ TypeMeta: metav1.TypeMeta{ @@ -89,7 +89,7 @@ func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, confi }, } - // create an appwrapper + // AppWrapper aw := &mcadv1beta1.AppWrapper{ ObjectMeta: metav1.ObjectMeta{ Name: "test-instascale", @@ -136,7 +136,5 @@ func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, confi }, } - _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - - return job, aw, err + return aw } diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 4c2cbab5..386b224d 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -22,6 +22,8 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -54,7 +56,10 @@ func TestInstascaleMachinePool(t *testing.T) { ShouldNot(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) // Setup batch job and AppWrapper - _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) + aw := instaScaleJobAppWrapper(test, namespace, cm) + + // apply AppWrapper to cluster + _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index 8e23d4e7..fad7717b 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -6,6 +6,8 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -29,13 +31,16 @@ func TestInstascaleMachineSet(t *testing.T) { "mnist.py": ReadFile(test, "mnist.py"), }) + // // Setup batch job and AppWrapper + aw := instaScaleJobAppWrapper(test, namespace, cm) + // look for machine set with aw name - expect to find it - test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale")))) + test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal(aw.Name)))) // look for machine belonging to the machine set, there should be none - test.Expect(GetMachines(test, "test-instascale")).Should(BeEmpty()) + test.Expect(GetMachines(test, aw.Name)).Should(BeEmpty()) - // // Setup batch job and AppWrapper - _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) + // apply AppWrapper to cluster + _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) // assert that AppWrapper goes to "Running" state @@ -43,13 +48,13 @@ func TestInstascaleMachineSet(t *testing.T) { Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) // look for machine belonging to the machine set - expect to find it - test.Eventually(Machines(test, "test-instascale"), TestTimeoutLong).Should(HaveLen(1)) + test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(HaveLen(1)) // assert that the AppWrapper goes to "Completed" state test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) // look for machine belonging to the machine set - there should be none - test.Eventually(Machines(test, "test-instascale"), TestTimeoutLong).Should(BeEmpty()) + test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(BeEmpty()) } From 5a0bb174966ac2530ce44f4dd54a8ea371c3bd88 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 17 Oct 2023 15:50:34 +0100 Subject: [PATCH 192/377] Replaced rayv1alpha1 with rayv1 --- test/e2e/mnist_raycluster_sdk_test.go | 4 ++-- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 24 +++++++++---------- test/support/ray.go | 22 ++++++++--------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index e9611065..29643fe6 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -21,7 +21,7 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -102,7 +102,7 @@ func TestMNISTRayClusterSDK(t *testing.T) { }, { Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1alpha1.GroupVersion.Group}, + APIGroups: []string{rayv1.GroupVersion.Group}, Resources: []string{"rayclusters", "rayclusters/status"}, }, { diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 7752d2ea..a7956f66 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -22,7 +22,7 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -60,18 +60,18 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.T().Logf("Created ConfigMap %s/%s successfully", mnist.Namespace, mnist.Name) // RayCluster - rayCluster := &rayv1alpha1.RayCluster{ + rayCluster := &rayv1.RayCluster{ TypeMeta: metav1.TypeMeta{ - APIVersion: rayv1alpha1.GroupVersion.String(), + APIVersion: rayv1.GroupVersion.String(), Kind: "RayCluster", }, ObjectMeta: metav1.ObjectMeta{ Name: "raycluster", Namespace: namespace.Name, }, - Spec: rayv1alpha1.RayClusterSpec{ + Spec: rayv1.RayClusterSpec{ RayVersion: GetRayVersion(), - HeadGroupSpec: rayv1alpha1.HeadGroupSpec{ + HeadGroupSpec: rayv1.HeadGroupSpec{ RayStartParams: map[string]string{ "dashboard-host": "0.0.0.0", }, @@ -135,7 +135,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, }, }, - WorkerGroupSpecs: []rayv1alpha1.WorkerGroupSpec{ + WorkerGroupSpecs: []rayv1.WorkerGroupSpec{ { Replicas: Ptr(int32(1)), MinReplicas: Ptr(int32(1)), @@ -220,16 +220,16 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - rayJob := &rayv1alpha1.RayJob{ + rayJob := &rayv1.RayJob{ TypeMeta: metav1.TypeMeta{ - APIVersion: rayv1alpha1.GroupVersion.String(), + APIVersion: rayv1.GroupVersion.String(), Kind: "RayJob", }, ObjectMeta: metav1.ObjectMeta{ Name: "mnist", Namespace: namespace.Name, }, - Spec: rayv1alpha1.RayJobSpec{ + Spec: rayv1.RayJobSpec{ Entrypoint: "python /home/ray/jobs/mnist.py", RuntimeEnv: base64.StdEncoding.EncodeToString([]byte(` { @@ -248,7 +248,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { ShutdownAfterJobFinishes: false, }, } - rayJob, err = test.Client().Ray().RayV1alpha1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) + rayJob, err = test.Client().Ray().RayV1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) @@ -262,9 +262,9 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.T().Logf("Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutLong). - Should(WithTransform(RayJobStatus, Satisfy(rayv1alpha1.IsJobTerminal))) + Should(WithTransform(RayJobStatus, Satisfy(rayv1.IsJobTerminal))) // Assert the Ray job has completed successfully test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). - To(WithTransform(RayJobStatus, Equal(rayv1alpha1.JobStatusSucceeded))) + To(WithTransform(RayJobStatus, Equal(rayv1.JobStatusSucceeded))) } diff --git a/test/support/ray.go b/test/support/ray.go index b69609ae..f18edaa0 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -18,27 +18,27 @@ package support import ( "github.com/onsi/gomega" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const RayJobDefaultClusterSelectorKey = "ray.io/cluster" -func RayJob(t Test, namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayJob { - return func(g gomega.Gomega) *rayv1alpha1.RayJob { - job, err := t.Client().Ray().RayV1alpha1().RayJobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) +func RayJob(t Test, namespace, name string) func(g gomega.Gomega) *rayv1.RayJob { + return func(g gomega.Gomega) *rayv1.RayJob { + job, err := t.Client().Ray().RayV1().RayJobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) return job } } -func GetRayJob(t Test, namespace, name string) *rayv1alpha1.RayJob { +func GetRayJob(t Test, namespace, name string) *rayv1.RayJob { t.T().Helper() return RayJob(t, namespace, name)(t) } -func RayJobStatus(job *rayv1alpha1.RayJob) rayv1alpha1.JobStatus { +func RayJobStatus(job *rayv1.RayJob) rayv1.JobStatus { return job.Status.JobStatus } @@ -48,20 +48,20 @@ func GetRayJobId(t Test, namespace, name string) string { return job.Status.JobId } -func RayCluster(t Test, namespace, name string) func(g gomega.Gomega) *rayv1alpha1.RayCluster { - return func(g gomega.Gomega) *rayv1alpha1.RayCluster { - cluster, err := t.Client().Ray().RayV1alpha1().RayClusters(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) +func RayCluster(t Test, namespace, name string) func(g gomega.Gomega) *rayv1.RayCluster { + return func(g gomega.Gomega) *rayv1.RayCluster { + cluster, err := t.Client().Ray().RayV1().RayClusters(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) g.Expect(err).NotTo(gomega.HaveOccurred()) return cluster } } -func GetRayCluster(t Test, namespace, name string) *rayv1alpha1.RayCluster { +func GetRayCluster(t Test, namespace, name string) *rayv1.RayCluster { t.T().Helper() return RayCluster(t, namespace, name)(t) } -func RayClusterState(cluster *rayv1alpha1.RayCluster) rayv1alpha1.ClusterState { +func RayClusterState(cluster *rayv1.RayCluster) rayv1.ClusterState { return cluster.Status.State } From 523d8e9fff5f89555061e91bd5addb69157371b3 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 17 Oct 2023 16:02:01 +0100 Subject: [PATCH 193/377] Added missing go files --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9166e25a..08f3cb31 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/instascale v0.1.0 github.com/project-codeflare/multi-cluster-app-dispatcher v1.36.0 - github.com/ray-project/kuberay/ray-operator v0.0.0-20231012011224-15ce568a0c36 + github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index f5d91a72..2c53de1c 100644 --- a/go.sum +++ b/go.sum @@ -420,8 +420,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/ray-project/kuberay/ray-operator v0.0.0-20231012011224-15ce568a0c36 h1:IzXHzI4OjL1ySpvmPHTWjFjmgBBjAZXBa3cQwejLPWY= -github.com/ray-project/kuberay/ray-operator v0.0.0-20231012011224-15ce568a0c36/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= +github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 h1:s+/WzpqoFot/fgo1uq7BLgeiOy3HZuT+UVtCb3h7R+I= +github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= From 06d724f053ba38b3c4e7d1f3e59868a293c9b17f Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 17 Oct 2023 16:08:16 +0100 Subject: [PATCH 194/377] Updated KubeRay version in MakeFile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 725b661a..4dcd3618 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v0.6.0 +KUBERAY_VERSION ?= v1.0.0-rc.1 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 From 86c12525b331eacf1d967aabb5bbb49f190e775c Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Tue, 17 Oct 2023 17:12:46 +0100 Subject: [PATCH 195/377] fix: update ubi image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index df237662..5c2df012 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.access.redhat.com/ubi8/go-toolset:1.19.10-16 as builder +FROM registry.access.redhat.io/ubi8/go-toolset:1.19 as builder WORKDIR /workspace # Copy the Go Modules manifests From 1268065a9b2f005f610ad1981ad4b4f5689e1f84 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Wed, 18 Oct 2023 10:18:40 +0100 Subject: [PATCH 196/377] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5c2df012..72dfe6e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.access.redhat.io/ubi8/go-toolset:1.19 as builder +FROM registry.access.redhat.com/ubi8/go-toolset:1.19 as builder WORKDIR /workspace # Copy the Go Modules manifests From 4b65c07d79f586557aac317f706fcc560b8fbeb7 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Thu, 19 Oct 2023 16:19:56 +0100 Subject: [PATCH 197/377] Updated MCAD controller cluster role for ingresses --- config/rbac/mcad-controller-ray-clusterrole.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/rbac/mcad-controller-ray-clusterrole.yaml b/config/rbac/mcad-controller-ray-clusterrole.yaml index f9562420..39ae487d 100644 --- a/config/rbac/mcad-controller-ray-clusterrole.yaml +++ b/config/rbac/mcad-controller-ray-clusterrole.yaml @@ -18,9 +18,9 @@ rules: - patch - delete - apiGroups: - - route.openshift.io + - networking.k8s.io resources: - - routes + - ingresses verbs: - create - delete From e10fae85668fd73991cd09ce39f636d80f18f637 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Thu, 19 Oct 2023 17:00:17 +0100 Subject: [PATCH 198/377] Re added routes to cluster role --- config/rbac/mcad-controller-ray-clusterrole.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config/rbac/mcad-controller-ray-clusterrole.yaml b/config/rbac/mcad-controller-ray-clusterrole.yaml index 39ae487d..18e3d98f 100644 --- a/config/rbac/mcad-controller-ray-clusterrole.yaml +++ b/config/rbac/mcad-controller-ray-clusterrole.yaml @@ -29,3 +29,15 @@ rules: - patch - update - watch +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch From 61ebcde8a5e70d0f33d705cbf7e6cb8b1f2a4b58 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Tue, 17 Oct 2023 12:03:37 +0100 Subject: [PATCH 199/377] ci: this calls the sync-fork workflow in the odh-cfo --- .github/workflows/odh-fork-sync.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/odh-fork-sync.yml diff --git a/.github/workflows/odh-fork-sync.yml b/.github/workflows/odh-fork-sync.yml new file mode 100644 index 00000000..72d4a366 --- /dev/null +++ b/.github/workflows/odh-fork-sync.yml @@ -0,0 +1,15 @@ +name: Call sync on OpenDataHub CFO fork sync +on: + release: + types: [published] + workflow_dispatch: + +jobs: + sync-fork: + runs-on: ubuntu-latest + steps: + - name: Sync-fork + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + run: | + gh workflow run sync-fork.yaml -R github.com/opendatahub-io/codeflare-operator -r main From 4b863f79d4b704c511ea404e27e3ec65d66b5a0f Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 20 Oct 2023 16:16:24 +0100 Subject: [PATCH 200/377] Copying MCAD CRDs in Kustomize config instead of referencing them (#357) * copying mcad crds in kustomize config instead of referencing them * addressing feedback * build: split crds into their own files * refactor: isntall yq if not exists * updating kustomization.yaml and verify_generated_files workflow * remove restore * adding crds * update check_yq target * Update Makefile * Update Makefile * Update Makefile * commenting out lines --------- Co-authored-by: Dimitri Saridakis --- .github/workflows/verify_generated_files.yml | 2 + Makefile | 39 +- config/crd/kustomization.yaml | 5 +- config/crd/mcad/kustomization.yaml | 2 +- config/crd/quota.codeflare_quotasubtrees.yaml | 122 +++ .../crd/workload.codeflare_appwrappers.yaml | 765 ++++++++++++++++++ .../workload.codeflare_schedulingspecs.yaml | 102 +++ 7 files changed, 1030 insertions(+), 7 deletions(-) create mode 100644 config/crd/quota.codeflare_quotasubtrees.yaml create mode 100644 config/crd/workload.codeflare_appwrappers.yaml create mode 100644 config/crd/workload.codeflare_schedulingspecs.yaml diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml index aa85fcff..b99a9a06 100644 --- a/.github/workflows/verify_generated_files.yml +++ b/.github/workflows/verify_generated_files.yml @@ -7,6 +7,7 @@ on: - '**.go' - '**go.mod' - '**go.sum' + - 'config/**' tags-ignore: - 'v*' pull_request: @@ -14,6 +15,7 @@ on: - '**.go' - '**go.mod' - '**go.sum' + - 'config/**' jobs: verify-imports: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 4dcd3618..00de9c19 100644 --- a/Makefile +++ b/Makefile @@ -146,9 +146,12 @@ defaults: gofmt -w $(DEFAULTS_TEST_FILE) +# this encounters sed issues on MacOS, quick fix is to use gsed or to escape the parentheses i.e. \( \) .PHONY: manifests -manifests: controller-gen ## Generate RBAC objects. +manifests: controller-gen kustomize ## Generate RBAC objects. $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." +# $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml +# $(KUSTOMIZE) build config/crd/mcad > config/crd/mcad.yaml && make split_yaml FILE=config/crd/mcad.yaml .PHONY: fmt fmt: ## Run go fmt against code. @@ -192,19 +195,16 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/crd | kubectl apply -f - git restore config/* .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - git restore config/* .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/${ENV} | kubectl apply -f - git restore config/* @@ -224,6 +224,7 @@ $(LOCALBIN): ## Tool Binaries KUSTOMIZE ?= $(LOCALBIN)/kustomize +YQ ?= $(LOCALBIN)/yq CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest OPENSHIFT-GOIMPORTS ?= $(LOCALBIN)/openshift-goimports @@ -284,7 +285,6 @@ validate-bundle: install-operator-sdk .PHONY: bundle bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion @@ -388,3 +388,32 @@ verify-imports: openshift-goimports ## Run import verifications. .PHONY: scorecard-bundle scorecard-bundle: install-operator-sdk ## Run scorecard tests on bundle image. $(OPERATOR_SDK) scorecard bundle + + +FILE ?= input.yaml # Default value, it isn't a file, but the make cmds fill hang for longer without it +temp_dir := temp_split +output_dir := 'config/crd/' + +.PHONY: check_yq +check_yq: + @command -v wget >/dev/null 2>&1 || (echo "Installing wget..."; apt-get install -y wget) + @command -v $(YQ) >/dev/null 2>&1 || (echo "Installing yq..."; wget https://github.com/mikefarah/yq/releases/download/v4.2.0/yq_linux_amd64.tar.gz -O - |\ + tar xz && mv yq_linux_amd64 $(YQ)) + +# this works on a MacOS by replacing awk with gawk +.PHONY: split_yaml +split_yaml: + @$(MAKE) check_yq + @mkdir -p $(temp_dir) + @awk '/apiVersion: /{if (x>0) close("$(temp_dir)/section_" x ".yaml"); x++}{print > "$(temp_dir)/section_"x".yaml"}' $(FILE) + @$(MAKE) process_sections + +.PHONY: process_sections +process_sections: + @mkdir -p $(output_dir) + @for section_file in $(temp_dir)/section_*; do \ + metadata_name=$$(YQ e '.metadata.name' $$section_file); \ + file_name=$$(echo $$metadata_name | awk -F'.' '{print $$2"."$$3"_"$$1".yaml"}'); \ + mv $$section_file $(output_dir)/$$file_name; \ + done + @rm -r $(temp_dir) diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 81d71c1e..73a57623 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,5 +2,8 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- mcad +- quota.codeflare_quotasubtrees.yaml +- workload.codeflare_appwrappers.yaml +- workload.codeflare_schedulingspecs.yaml + #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index 8ea96b05..e016d77f 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v0.0.0 # kpt-set: ${MCAD_CRD} +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.36.0 diff --git a/config/crd/quota.codeflare_quotasubtrees.yaml b/config/crd/quota.codeflare_quotasubtrees.yaml new file mode 100644 index 00000000..44bb5e0a --- /dev/null +++ b/config/crd/quota.codeflare_quotasubtrees.yaml @@ -0,0 +1,122 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: quotasubtrees.quota.codeflare.dev +spec: + group: quota.codeflare.dev + names: + kind: QuotaSubtree + listKind: QuotaSubtreeList + plural: quotasubtrees + singular: quotasubtree + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: QuotaSubtree is a specification for a quota subtree resource + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: QuotaSubtreeSpec is the spec for a resource plan + properties: + children: + items: + description: Child is the spec for a QuotaSubtree resource + properties: + name: + type: string + namespace: + type: string + path: + type: string + quotas: + description: Quota is the spec for a QuotaSubtree resource + properties: + disabled: + type: boolean + hardLimit: + type: boolean + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: array + parent: + type: string + parentNamespace: + type: string + type: object + status: + description: QuotaSubtreeStatus is the status for a QuotaSubtree resource + properties: + children: + items: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child + resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + type: array + totalAllocation: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child + resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + required: + - children + - totalAllocation + type: object + required: + - spec + type: object + served: true + storage: true +--- diff --git a/config/crd/workload.codeflare_appwrappers.yaml b/config/crd/workload.codeflare_appwrappers.yaml new file mode 100644 index 00000000..e1d26041 --- /dev/null +++ b/config/crd/workload.codeflare_appwrappers.yaml @@ -0,0 +1,765 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: appwrappers.workload.codeflare.dev +spec: + group: workload.codeflare.dev + names: + kind: AppWrapper + listKind: AppWrapperList + plural: appwrappers + singular: appwrapper + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Definition of AppWrapper class + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AppWrapperSpec describes how the App Wrapper will look like. + properties: + priority: + format: int32 + type: integer + priorityslope: + format: float + type: number + resources: + description: a collection of AppWrapperResource + properties: + GenericItems: + items: + description: AppWrapperGenericResource is App Wrapper aggregation + resource + properties: + allocated: + description: The number of allocated replicas from this + resource type + format: int32 + type: integer + completionstatus: + description: 'Optional field that drives completion status + of this AppWrapper. This field within an item of an AppWrapper + determines the full state of the AppWrapper. The completionstatus + field contains a list of conditions that make the associate + item considered completed, for instance: - completion + conditions could be "Complete" or "Failed". The associated + item''s level .status.conditions[].type field is monitored + for any one of these conditions. Once all items with this + option is set and the conditionstatus is met the entire + AppWrapper state will be changed to one of the valid AppWrapper + completion state. Note: - this is an AND operation for + all items where this option is set. See the list of AppWrapper + states for a list of valid complete states.' + type: string + custompodresources: + description: Optional section that specifies resource requirements + for non-standard k8s resources, follows same format as + that of standard k8s resources. + items: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, + quantity) pairs. + type: object + replicas: + type: integer + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'todo: replace with Containers []Container + Contain v1.ResourceRequirements' + type: object + required: + - replicas + - requests + type: object + type: array + generictemplate: + description: The template for the resource; it is now a + raw text because we don't know for what resource it should + be instantiated + type: object + x-kubernetes-embedded-resource: true + x-kubernetes-preserve-unknown-fields: true + minavailable: + description: The minimal available pods to run for this + AppWrapper; the default value is nil + format: int32 + type: integer + priority: + description: The priority of this resource + format: int32 + type: integer + priorityslope: + description: The increasing rate of priority value for this + resource + format: float + type: number + replicas: + description: Replicas is the number of desired replicas + format: int32 + type: integer + type: object + type: array + type: object + schedulingSpec: + description: SchedSpec specifies the parameters used for scheduling + generic items wrapped inside AppWrappers. It defines the policy + for requeuing jobs based on the number of running pods. + properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object + minAvailable: + description: Expected number of pods in running and/or completed + state. Requeuing is triggered when the number of running/completed + pods is not equal to this value. When not specified, requeuing + is disabled and no check is performed. + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + requeuing: + description: Specification of the requeuing strategy based on + waiting time. Values in this field control how often the pod + check should happen, and if requeuing has reached its maximum + number of times. + properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time + between requeuing checks. The values available are 'exponential', + 'linear', or 'none'. For example, 'exponential' growth would + double the 'timeInSeconds' value every time a requeuing + event is triggered. If the string value is misspelled or + not one of the possible options, the growth behavior is + defaulted to 'none'. + type: string + initialTimeInSeconds: + description: Value to keep track of the initial wait time. + Users cannot set this as it is taken from 'timeInSeconds'. + type: integer + maxNumRequeuings: + default: 0 + description: Maximum number of requeuing events allowed. Once + this value is reached (e.g., 'numRequeuings = maxNumRequeuings', + no more requeuing checks are performed and the generic items + are stopped and removed from the cluster (AppWrapper remains + deployed). + type: integer + maxTimeInSeconds: + default: 0 + description: Maximum waiting time for requeuing checks. + type: integer + numRequeuings: + default: 0 + description: Field to keep track of how many times a requeuing + event has been triggered. + type: integer + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions + are checked. This value is specified by the user, but it + may grow as requeuing events happen. + type: integer + type: object + type: object + selector: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + service: + description: AppWrapperService is App Wrapper service definition + properties: + spec: + description: ServiceSpec describes the attributes that a user + creates on a service. + properties: + allocateLoadBalancerNodePorts: + description: allocateLoadBalancerNodePorts defines if NodePorts + will be automatically allocated for services with type LoadBalancer. Default + is "true". It may be set to "false" if the cluster load-balancer + does not rely on NodePorts. If the caller requests specific + NodePorts (by specifying a value), those requests will be + respected, regardless of this field. This field may only + be set for services with type LoadBalancer and will be cleared + if the type is changed to any other type. + type: boolean + clusterIP: + description: 'clusterIP is the IP address of the service and + is usually assigned randomly. If an address is specified + manually, is in-range (as per system configuration), and + is not in use, it will be allocated to the service; otherwise + creation of the service will fail. This field may not be + changed through updates unless the type field is also being + changed to ExternalName (which requires this field to be + blank) or the type field is being changed from ExternalName + (in which case this field may optionally be specified, as + describe above). Valid values are "None", empty string + (""), or a valid IP address. Setting this to "None" makes + a "headless service" (no virtual IP), which is useful when + direct endpoint connections are preferred and proxying is + not required. Only applies to types ClusterIP, NodePort, + and LoadBalancer. If this field is specified when creating + a Service of type ExternalName, creation will fail. This + field will be wiped when updating a Service to type ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + clusterIPs: + description: "ClusterIPs is a list of IP addresses assigned + to this service, and are usually assigned randomly. If + an address is specified manually, is in-range (as per system + configuration), and is not in use, it will be allocated + to the service; otherwise creation of the service will fail. + This field may not be changed through updates unless the + type field is also being changed to ExternalName (which + requires this field to be empty) or the type field is being + changed from ExternalName (in which case this field may + optionally be specified, as describe above). Valid values + are \"None\", empty string (\"\"), or a valid IP address. + \ Setting this to \"None\" makes a \"headless service\" + (no virtual IP), which is useful when direct endpoint connections + are preferred and proxying is not required. Only applies + to types ClusterIP, NodePort, and LoadBalancer. If this + field is specified when creating a Service of type ExternalName, + creation will fail. This field will be wiped when updating + a Service to type ExternalName. If this field is not specified, + it will be initialized from the clusterIP field. If this + field is specified, clients must ensure that clusterIPs[0] + and clusterIP have the same value. \n This field may hold + a maximum of two entries (dual-stack IPs, in either order). + These IPs must correspond to the values of the ipFamilies + field. Both clusterIPs and ipFamilies are governed by the + ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this service. These + IPs are not managed by Kubernetes. The user is responsible + for ensuring that traffic arrives at a node with this IP. A + common example is external load-balancers that are not part + of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that discovery + mechanisms will return as an alias for this service (e.g. + a DNS CNAME record). No proxying will be involved. Must + be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: externalTrafficPolicy describes how nodes distribute + service traffic they receive on one of the Service's "externally-facing" + addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). + If set to "Local", the proxy will configure the service + in a way that assumes that external load balancers will + take care of balancing the service traffic between nodes, + and so each node will deliver traffic only to the node-local + endpoints of the service, without masquerading the client + source IP. (Traffic mistakenly sent to a node with no endpoints + will be dropped.) The default value, "Cluster", uses the + standard behavior of routing to all endpoints evenly (possibly + modified by topology and other features). Note that traffic + sent to an External IP or LoadBalancer IP from within the + cluster will always get "Cluster" semantics, but clients + sending to a NodePort from within the cluster may need to + take traffic policy into account when picking a node. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. This only applies when type is + set to LoadBalancer and externalTrafficPolicy is set to + Local. If a value is specified, is in-range, and is not + in use, it will be used. If not specified, a value will + be automatically allocated. External systems (e.g. load-balancers) + can use this port to determine if a given node holds endpoints + for this service or not. If this field is specified when + creating a Service which does not need it, creation will + fail. This field will be wiped when updating a Service to + no longer need it (e.g. changing type). This field cannot + be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: InternalTrafficPolicy describes how nodes distribute + service traffic they receive on the ClusterIP. If set to + "Local", the proxy will assume that pods only want to talk + to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The + default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology + and other features). + type: string + ipFamilies: + description: "IPFamilies is a list of IP families (e.g. IPv4, + IPv6) assigned to this service. This field is usually assigned + automatically based on cluster configuration and the ipFamilyPolicy + field. If this field is specified manually, the requested + family is available in the cluster, and ipFamilyPolicy allows + it, it will be used; otherwise creation of the service will + fail. This field is conditionally mutable: it allows for + adding or removing a secondary IP family, but it does not + allow changing the primary IP family of the Service. Valid + values are \"IPv4\" and \"IPv6\". This field only applies + to Services of types ClusterIP, NodePort, and LoadBalancer, + and does apply to \"headless\" services. This field will + be wiped when updating a Service to type ExternalName. \n + This field may hold a maximum of two entries (dual-stack + families, in either order). These families must correspond + to the values of the clusterIPs field, if specified. Both + clusterIPs and ipFamilies are governed by the ipFamilyPolicy + field." + items: + description: IPFamily represents the IP Family (IPv4 or + IPv6). This type is used to express the family of an IP + expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: IPFamilyPolicy represents the dual-stack-ness + requested or required by this Service. If there is no value + provided, then this field will be set to SingleStack. Services + can be "SingleStack" (a single IP family), "PreferDualStack" + (two IP families on dual-stack configured clusters or a + single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise + fail). The ipFamilies and clusterIPs fields depend on the + value of this field. This field will be wiped when updating + a service to type ExternalName. + type: string + loadBalancerClass: + description: loadBalancerClass is the class of the load balancer + implementation this Service belongs to. If specified, the + value of this field must be a label-style identifier, with + an optional prefix, e.g. "internal-vip" or "example.com/internal-vip". + Unprefixed names are reserved for end-users. This field + can only be set when the Service type is 'LoadBalancer'. + If not set, the default load balancer implementation is + used, today this is typically done through the cloud provider + integration, but should apply for any default implementation. + If set, it is assumed that a load balancer implementation + is watching for Services with a matching class. Any default + load balancer implementation (e.g. cloud providers) should + ignore Services that set this field. This field can only + be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped + when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer. + This feature depends on whether the underlying cloud-provider + supports specifying the loadBalancerIP when a load balancer + is created. This field will be ignored if the cloud-provider + does not support the feature. Deprecated: This field was + under-specified and its meaning varies across implementations, + and it cannot support dual-stack. As of Kubernetes v1.24, + users are encouraged to use implementation-specific annotations + when available. This field may be removed in a future API + version.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider load-balancer + will be restricted to the specified client IPs. This field + will be ignored if the cloud-provider does not support the + feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this service. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + appProtocol: + description: The application protocol for this port. + This field follows standard Kubernetes label syntax. + Un-prefixed names are reserved for IANA standard service + names (as per RFC-6335 and https://www.iana.org/assignments/service-names). + Non-standard protocols should use prefixed names such + as mycompany.com/my-custom-protocol. + type: string + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field in + the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this service + is exposed when type is NodePort or LoadBalancer. Usually + assigned by the system. If a value is specified, in-range, + and not in use it will be used, otherwise the operation + will fail. If not specified, a port will be allocated + if this Service requires one. If this field is specified + when creating a Service which does not need it, creation + will fail. This field will be wiped when updating + a Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this service. + format: int32 + type: integer + protocol: + default: TCP + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access on + the pods targeted by the service. Number must be in + the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored for + services with clusterIP=None, and should be omitted + or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: publishNotReadyAddresses indicates that any agent + which deals with endpoints for this Service should disregard + any indications of ready/not-ready. The primary use case + for setting this field is for a StatefulSet's Headless Service + to propagate SRV DNS records for its Pods for the purpose + of peer discovery. The Kubernetes controllers that generate + Endpoints and EndpointSlice resources for Services interpret + this to mean that all endpoints are considered "ready" even + if the Pods themselves are not. Agents which consume only + Kubernetes generated endpoints through the Endpoints or + EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only applies + to types ClusterIP, NodePort, and LoadBalancer. Ignored + if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of Client + IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, + NodePort, and LoadBalancer. "ClusterIP" allocates a cluster-internal + IP address for load-balancing to endpoints. Endpoints are + determined by the selector or if that is not specified, + by manual construction of an Endpoints object or EndpointSlice + objects. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints rather + than a virtual IP. "NodePort" builds on ClusterIP and allocates + a port on every node which routes to the same endpoints + as the clusterIP. "LoadBalancer" builds on NodePort and + creates an external load-balancer (if supported in the current + cloud) which routes to the same endpoints as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + required: + - spec + type: object + required: + - resources + type: object + status: + description: AppWrapperStatus represents the current state of a AppWrapper + properties: + Succeeded: + description: The number of resources which reached phase Succeeded. + format: int32 + type: integer + canrun: + description: Can run? + type: boolean + conditions: + description: Represents the latest available observations of the AppWrapper's + current condition. + items: + description: AppWrapperCondition describes the state of an AppWrapper + at a certain point. + properties: + lastTransitionMicroTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateMicroTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of AppWrapper condition. + type: string + required: + - status + - type + type: object + type: array + controllerfirstdispatchtimestamp: + description: Microsecond level timestamp when controller first dispatches + the AppWrapper + format: date-time + type: string + controllerfirsttimestamp: + description: Microsecond level timestamp when controller first sees + QueueJob (by Informer) + format: date-time + type: string + failed: + description: The number of resources which reached phase Failed. + format: int32 + type: integer + filterignore: + description: Tell Informer to ignore this update message (do not generate + a controller event) + type: boolean + isdispatched: + description: Is Dispatched? + type: boolean + local: + description: Indicate if message is a duplicate (for Informer to recognize + duplicate messages) + type: boolean + message: + type: string + numberOfRequeueings: + description: Field to keep track of how many times a requeuing event + has been triggered + type: integer + pending: + description: The number of pending pods. + format: int32 + type: integer + pendingpodconditions: + description: Represents the latest available observations of pods + belonging to the AppWrapper. + items: + properties: + conditions: + items: + description: PodCondition contains details for the current + condition of this pod. + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned from + one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the + condition's last transition. + type: string + status: + description: 'Status is the status of the condition. Can + be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + type: + description: 'Type is the type of the condition. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + required: + - status + - type + type: object + type: array + podname: + type: string + type: object + type: array + queuejobstate: + description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, + ... + type: string + requeueingTimeInSeconds: + description: Field to keep track of total number of seconds spent + in requeueing + type: integer + running: + format: int32 + type: integer + sender: + description: Indicate sender of this message (extremely useful for + debugging) + type: string + state: + description: State - Pending, Running, Failed, Deleted + type: string + systempriority: + description: System defined Priority + format: float + type: number + template: + description: The minimal available resources to run for this AppWrapper + (is this different from the MinAvailable from JobStatus) + format: int32 + type: integer + totalcpu: + description: The number of CPU consumed by all pods belonging to the + AppWrapper. + format: int32 + type: integer + totalgpu: + description: The total number of GPUs consumed by all pods belonging + to the AppWrapper. + format: int32 + type: integer + totalmemory: + description: The amount of memory consumed by all pods belonging to + the AppWrapper. + format: int32 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- diff --git a/config/crd/workload.codeflare_schedulingspecs.yaml b/config/crd/workload.codeflare_schedulingspecs.yaml new file mode 100644 index 00000000..fc44a432 --- /dev/null +++ b/config/crd/workload.codeflare_schedulingspecs.yaml @@ -0,0 +1,102 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: schedulingspecs.workload.codeflare.dev +spec: + group: workload.codeflare.dev + names: + kind: SchedulingSpec + listKind: SchedulingSpecList + plural: schedulingspecs + singular: schedulingspec + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object + minAvailable: + description: Expected number of pods in running and/or completed state. + Requeuing is triggered when the number of running/completed pods + is not equal to this value. When not specified, requeuing is disabled + and no check is performed. + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + requeuing: + description: Specification of the requeuing strategy based on waiting + time. Values in this field control how often the pod check should + happen, and if requeuing has reached its maximum number of times. + properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time between + requeuing checks. The values available are 'exponential', 'linear', + or 'none'. For example, 'exponential' growth would double the + 'timeInSeconds' value every time a requeuing event is triggered. + If the string value is misspelled or not one of the possible + options, the growth behavior is defaulted to 'none'. + type: string + initialTimeInSeconds: + description: Value to keep track of the initial wait time. Users + cannot set this as it is taken from 'timeInSeconds'. + type: integer + maxNumRequeuings: + default: 0 + description: Maximum number of requeuing events allowed. Once + this value is reached (e.g., 'numRequeuings = maxNumRequeuings', + no more requeuing checks are performed and the generic items + are stopped and removed from the cluster (AppWrapper remains + deployed). + type: integer + maxTimeInSeconds: + default: 0 + description: Maximum waiting time for requeuing checks. + type: integer + numRequeuings: + default: 0 + description: Field to keep track of how many times a requeuing + event has been triggered. + type: integer + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions + are checked. This value is specified by the user, but it may + grow as requeuing events happen. + type: integer + type: object + type: object + required: + - metadata + type: object + served: true + storage: true From 9d8297e81e677e7465166791230050b8ae2202bf Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Fri, 20 Oct 2023 15:52:28 +0000 Subject: [PATCH 201/377] Update dependency versions for release v1.0.0-rc.3 --- Makefile | 6 +++--- README.md | 8 ++++---- go.mod | 4 ++-- go.sum | 8 ++++---- test/support/defaults.go | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 00de9c19..636398ea 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.1.0 +INSTASCALE_VERSION ?= v0.2.0 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.36.0 +MCAD_VERSION ?= v1.37.0 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} @@ -28,7 +28,7 @@ KUBERAY_VERSION ?= v1.0.0-rc.1 RAY_VERSION ?= 2.5.0 # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK -CODEFLARE_SDK_VERSION ?= 0.9.0 +CODEFLARE_SDK_VERSION ?= 0.10.0 # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to diff --git a/README.md b/README.md index b2f40931..aff1b9a6 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.0.0-rc.2](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.2) | -| Multi-Cluster App Dispatcher | [v1.36.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.36.0) | -| CodeFlare-SDK | [v0.9.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.9.0) | -| InstaScale | [v0.1.0](https://github.com/project-codeflare/instascale/releases/tag/v0.1.0) | +| CodeFlare Operator | [v1.0.0-rc.3](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.3) | +| Multi-Cluster App Dispatcher | [v1.37.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.37.0) | +| CodeFlare-SDK | [v0.10.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.10.0) | +| InstaScale | [v0.2.0](https://github.com/project-codeflare/instascale/releases/tag/v0.2.0) | | KubeRay | [v0.5.0](https://github.com/ray-project/kuberay/releases/tag/v0.5.0) | diff --git a/go.mod b/go.mod index 08f3cb31..ffda7763 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.368 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/instascale v0.1.0 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.36.0 + github.com/project-codeflare/instascale v0.2.0 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 diff --git a/go.sum b/go.sum index 2c53de1c..06269a9f 100644 --- a/go.sum +++ b/go.sum @@ -389,10 +389,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/instascale v0.1.0 h1:oP5PwUIDvh+D3fbg8sT9dKphl9K/Tb/RraTFQezLOxI= -github.com/project-codeflare/instascale v0.1.0/go.mod h1:c4I8w4R0I8OJdkwA/ht/o/3lxRCBx4YdFJ3D1DulFVA= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.36.0 h1:QSc16Kd4OBidGY4rQivMrUX9B9cNKttfZIRKsR4BX9E= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.36.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/instascale v0.2.0 h1:85Bbz6AsAYIKROqFmDIGS9hWV6pDf1iTizDVipxJwGg= +github.com/project-codeflare/instascale v0.2.0/go.mod h1:6x6WhDZiJRRwIAK7sAND3YYSGU3ry2Wa/RqyaHXKy18= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 h1:oyhdLdc4BgA4zcH1zlRrSrYpzuVxV5QLDbyIXrwnQqs= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= diff --git a/test/support/defaults.go b/test/support/defaults.go index 1beb7e89..bf9db081 100644 --- a/test/support/defaults.go +++ b/test/support/defaults.go @@ -5,7 +5,7 @@ package support // *********************** const ( - CodeFlareSDKVersion = "0.9.0" + CodeFlareSDKVersion = "0.10.0" RayVersion = "2.5.0" RayImage = "rayproject/ray:2.5.0" ) From 9db28cf1d3577a703610e5c065907e529c36abe2 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Sat, 21 Oct 2023 19:15:11 +0100 Subject: [PATCH 202/377] build: pin ubi image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 72dfe6e7..ab721099 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.access.redhat.com/ubi8/go-toolset:1.19 as builder +FROM registry.access.redhat.com/ubi8/go-toolset:1.19.13 as builder WORKDIR /workspace # Copy the Go Modules manifests From 6f7865b6dfdc46da3d57dc6c60991790ef274b7a Mon Sep 17 00:00:00 2001 From: James Busche Date: Mon, 16 Oct 2023 15:03:09 -0700 Subject: [PATCH 203/377] fixing golang items in manager Signed-off-by: James Busche --- go.mod | 12 ++++++------ go.sum | 23 ++++++++++++----------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index ffda7763..8c43cbbc 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.1 // indirect + github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -92,13 +92,13 @@ require ( go.opentelemetry.io/otel/trace v1.10.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.11.0 // indirect - golang.org/x/net v0.12.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect golang.org/x/sync v0.2.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/term v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 06269a9f..fe0d21e8 100644 --- a/go.sum +++ b/go.sum @@ -153,8 +153,9 @@ github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= +github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -541,8 +542,8 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -619,8 +620,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -702,13 +703,13 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -718,8 +719,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From f35ffa2bd85f6fe891dab38ca8616aaf64d91b34 Mon Sep 17 00:00:00 2001 From: James Busche Date: Thu, 19 Oct 2023 15:27:46 -0700 Subject: [PATCH 204/377] Update to opentelemetry Signed-off-by: James Busche --- go.mod | 39 +++++++++++----------- go.sum | 100 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 69 insertions(+), 70 deletions(-) diff --git a/go.mod b/go.mod index 8c43cbbc..83c2243d 100644 --- a/go.mod +++ b/go.mod @@ -28,8 +28,8 @@ require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -47,7 +47,7 @@ require ( github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect - github.com/golang/glog v1.0.0 // indirect + github.com/golang/glog v1.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/cel-go v0.12.6 // indirect @@ -57,7 +57,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -81,30 +81,31 @@ require ( go.etcd.io/etcd/api/v3 v3.5.5 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect go.etcd.io/etcd/client/v3 v3.5.5 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 // indirect - go.opentelemetry.io/otel v1.10.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect - go.opentelemetry.io/otel/metric v0.31.0 // indirect - go.opentelemetry.io/otel/sdk v1.10.0 // indirect - go.opentelemetry.io/otel/trace v1.10.0 // indirect - go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/sdk v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/sync v0.2.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect - google.golang.org/grpc v1.49.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/grpc v1.58.2 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index fe0d21e8..5f4d6788 100644 --- a/go.sum +++ b/go.sum @@ -13,13 +13,15 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.97.0 h1:3DXvAyifywvq64LfkKaMOmkWPS1CikIQdMe2lY9vxU8= +cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -57,25 +59,24 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -103,8 +104,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -157,8 +158,9 @@ github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -205,7 +207,6 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -236,8 +237,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= @@ -325,8 +326,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -425,6 +426,7 @@ github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 h github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -458,7 +460,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= @@ -487,34 +489,32 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 h1:Ajldaqhxqw/gNzQA45IKFWLdG7jZuXX/wBW1d5qvbUI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= -go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= -go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 h1:TaB+1rQhddO1sF71MpZOZAuSPW1klK2M8XxfrBMfK7Y= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 h1:pDDYmo0QadUPal5fwXoY1pmMpFcdyhXOmL5drCrI3vU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 h1:KtiUEhQmj/Pa874bVYKGNVdq8NPKiacPbaRRtgXi+t4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= -go.opentelemetry.io/otel/metric v0.31.0 h1:6SiklT+gfWAwWUR0meEMxQBtihpiEs4c+vL9spDTqUs= -go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= -go.opentelemetry.io/otel/sdk v1.10.0 h1:jZ6K7sVn04kk/3DNUdJ4mqRlGDiXAVuIG+MMENpTNdY= -go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= -go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= -go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 h1:RsQi0qJ2imFfCvZabqzM9cNXBG8k6gXMv1A0cXRmH6A= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0/go.mod h1:vsh3ySueQCiKPxFLvjWC4Z135gIa34TQ/NSqkDTZYUM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -628,10 +628,9 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -643,8 +642,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -688,7 +687,6 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -841,10 +839,13 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -862,10 +863,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -879,9 +878,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 9912d96ccc33eec50061df248f4cfa4a6b4c934a Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Mon, 23 Oct 2023 20:42:23 +0000 Subject: [PATCH 205/377] Update dependency versions for release v1.0.0-rc.4 --- Makefile | 6 +++--- README.md | 8 ++++---- go.mod | 4 ++-- go.sum | 8 ++++---- test/support/defaults.go | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 636398ea..e7bd3787 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.2.0 +INSTASCALE_VERSION ?= v0.2.1 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.37.0 +MCAD_VERSION ?= v1.37.1 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} @@ -28,7 +28,7 @@ KUBERAY_VERSION ?= v1.0.0-rc.1 RAY_VERSION ?= 2.5.0 # CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK -CODEFLARE_SDK_VERSION ?= 0.10.0 +CODEFLARE_SDK_VERSION ?= 0.10.1 # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to diff --git a/README.md b/README.md index aff1b9a6..a21c1647 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.0.0-rc.3](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.3) | -| Multi-Cluster App Dispatcher | [v1.37.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.37.0) | -| CodeFlare-SDK | [v0.10.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.10.0) | -| InstaScale | [v0.2.0](https://github.com/project-codeflare/instascale/releases/tag/v0.2.0) | +| CodeFlare Operator | [v1.0.0-rc.4](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.4) | +| Multi-Cluster App Dispatcher | [v1.37.1](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.37.1) | +| CodeFlare-SDK | [v0.10.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.10.1) | +| InstaScale | [v0.2.1](https://github.com/project-codeflare/instascale/releases/tag/v0.2.1) | | KubeRay | [v0.5.0](https://github.com/ray-project/kuberay/releases/tag/v0.5.0) | diff --git a/go.mod b/go.mod index 83c2243d..40254116 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,8 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.368 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c - github.com/project-codeflare/instascale v0.2.0 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 + github.com/project-codeflare/instascale v0.2.1 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 diff --git a/go.sum b/go.sum index 5f4d6788..867260a3 100644 --- a/go.sum +++ b/go.sum @@ -391,10 +391,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/instascale v0.2.0 h1:85Bbz6AsAYIKROqFmDIGS9hWV6pDf1iTizDVipxJwGg= -github.com/project-codeflare/instascale v0.2.0/go.mod h1:6x6WhDZiJRRwIAK7sAND3YYSGU3ry2Wa/RqyaHXKy18= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 h1:oyhdLdc4BgA4zcH1zlRrSrYpzuVxV5QLDbyIXrwnQqs= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/instascale v0.2.1 h1:t+Ax/sk4yPQznO6N+U8Cq+bk3afCcZDj9wnHfiGSDBg= +github.com/project-codeflare/instascale v0.2.1/go.mod h1:zSzBTP4cFkg+bD4JyYTDmDnGwVKY/+6ACks57NAiscc= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 h1:hZhGwKTPeHYYhNbvO27NOjozVpy7m3I3apKf81u9U3A= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= diff --git a/test/support/defaults.go b/test/support/defaults.go index bf9db081..2e0227cf 100644 --- a/test/support/defaults.go +++ b/test/support/defaults.go @@ -5,7 +5,7 @@ package support // *********************** const ( - CodeFlareSDKVersion = "0.10.0" + CodeFlareSDKVersion = "0.10.1" RayVersion = "2.5.0" RayImage = "rayproject/ray:2.5.0" ) From c7954a9bd6b19f5978bcf318ebb7d839ad4b1b99 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Sun, 22 Oct 2023 20:22:09 +0100 Subject: [PATCH 206/377] updating make manifests target --- Makefile | 43 +- config/crd/crd-appwrapper.yml | 462 +++++++++++ config/crd/crd-quotasubtree.yml | 115 +++ config/crd/crd-schedulingspec.yml | 80 ++ config/crd/kustomization.yaml | 6 +- config/crd/mcad/kustomization.yaml | 2 +- config/crd/quota.codeflare_quotasubtrees.yaml | 122 --- .../crd/workload.codeflare_appwrappers.yaml | 765 ------------------ .../workload.codeflare_schedulingspecs.yaml | 102 --- 9 files changed, 672 insertions(+), 1025 deletions(-) create mode 100644 config/crd/crd-appwrapper.yml create mode 100644 config/crd/crd-quotasubtree.yml create mode 100644 config/crd/crd-schedulingspec.yml delete mode 100644 config/crd/quota.codeflare_quotasubtrees.yaml delete mode 100644 config/crd/workload.codeflare_appwrappers.yaml delete mode 100644 config/crd/workload.codeflare_schedulingspecs.yaml diff --git a/Makefile b/Makefile index e7bd3787..d89caf1c 100644 --- a/Makefile +++ b/Makefile @@ -148,10 +148,12 @@ defaults: # this encounters sed issues on MacOS, quick fix is to use gsed or to escape the parentheses i.e. \( \) .PHONY: manifests -manifests: controller-gen kustomize ## Generate RBAC objects. +manifests: controller-gen kustomize install-yq ## Generate RBAC objects. $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." -# $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml -# $(KUSTOMIZE) build config/crd/mcad > config/crd/mcad.yaml && make split_yaml FILE=config/crd/mcad.yaml + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml + $(KUSTOMIZE) build config/crd/mcad > config/crd/mcad.yaml + $(YQ) -s '"crd-" + .spec.names.singular' config/crd/mcad.yaml --no-doc + mv crd-*.yml config/crd .PHONY: fmt fmt: ## Run go fmt against code. @@ -236,6 +238,7 @@ SED ?= /usr/bin/sed KUSTOMIZE_VERSION ?= v4.5.4 CODEGEN_VERSION ?= v0.27.2 CONTROLLER_TOOLS_VERSION ?= v0.9.2 +YQ_VERSION ?= v4.30.8 ## latest version that works with go1.19 OPERATOR_SDK_VERSION ?= v1.27.0 GH_CLI_VERSION ?= 2.30.0 @@ -261,6 +264,11 @@ controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessar $(CONTROLLER_GEN): $(LOCALBIN) test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) +.PHONY: install-yq +install-yq: $(YQ) ## Download yq locally if necessary +$(YQ): $(LOCALBIN) + test -s $(LOCALBIN)/yq || GOBIN=$(LOCALBIN) go install github.com/mikefarah/yq/v4@$(YQ_VERSION) + .PHONY: envtest envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) @@ -388,32 +396,3 @@ verify-imports: openshift-goimports ## Run import verifications. .PHONY: scorecard-bundle scorecard-bundle: install-operator-sdk ## Run scorecard tests on bundle image. $(OPERATOR_SDK) scorecard bundle - - -FILE ?= input.yaml # Default value, it isn't a file, but the make cmds fill hang for longer without it -temp_dir := temp_split -output_dir := 'config/crd/' - -.PHONY: check_yq -check_yq: - @command -v wget >/dev/null 2>&1 || (echo "Installing wget..."; apt-get install -y wget) - @command -v $(YQ) >/dev/null 2>&1 || (echo "Installing yq..."; wget https://github.com/mikefarah/yq/releases/download/v4.2.0/yq_linux_amd64.tar.gz -O - |\ - tar xz && mv yq_linux_amd64 $(YQ)) - -# this works on a MacOS by replacing awk with gawk -.PHONY: split_yaml -split_yaml: - @$(MAKE) check_yq - @mkdir -p $(temp_dir) - @awk '/apiVersion: /{if (x>0) close("$(temp_dir)/section_" x ".yaml"); x++}{print > "$(temp_dir)/section_"x".yaml"}' $(FILE) - @$(MAKE) process_sections - -.PHONY: process_sections -process_sections: - @mkdir -p $(output_dir) - @for section_file in $(temp_dir)/section_*; do \ - metadata_name=$$(YQ e '.metadata.name' $$section_file); \ - file_name=$$(echo $$metadata_name | awk -F'.' '{print $$2"."$$3"_"$$1".yaml"}'); \ - mv $$section_file $(output_dir)/$$file_name; \ - done - @rm -r $(temp_dir) diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml new file mode 100644 index 00000000..9140857a --- /dev/null +++ b/config/crd/crd-appwrapper.yml @@ -0,0 +1,462 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: appwrappers.workload.codeflare.dev +spec: + group: workload.codeflare.dev + names: + kind: AppWrapper + listKind: AppWrapperList + plural: appwrappers + singular: appwrapper + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Definition of AppWrapper class + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AppWrapperSpec describes how the App Wrapper will look like. + properties: + priority: + format: int32 + type: integer + priorityslope: + format: float + type: number + resources: + description: a collection of AppWrapperResource + properties: + GenericItems: + items: + description: AppWrapperGenericResource is App Wrapper aggregation resource + properties: + allocated: + description: The number of allocated replicas from this resource type + format: int32 + type: integer + completionstatus: + description: 'Optional field that drives completion status of this AppWrapper. This field within an item of an AppWrapper determines the full state of the AppWrapper. The completionstatus field contains a list of conditions that make the associate item considered completed, for instance: - completion conditions could be "Complete" or "Failed". The associated item''s level .status.conditions[].type field is monitored for any one of these conditions. Once all items with this option is set and the conditionstatus is met the entire AppWrapper state will be changed to one of the valid AppWrapper completion state. Note: - this is an AND operation for all items where this option is set. See the list of AppWrapper states for a list of valid complete states.' + type: string + custompodresources: + description: Optional section that specifies resource requirements for non-standard k8s resources, follows same format as that of standard k8s resources. + items: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) pairs. + type: object + replicas: + type: integer + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'todo: replace with Containers []Container Contain v1.ResourceRequirements' + type: object + required: + - replicas + - requests + type: object + type: array + generictemplate: + description: The template for the resource; it is now a raw text because we don't know for what resource it should be instantiated + type: object + x-kubernetes-embedded-resource: true + x-kubernetes-preserve-unknown-fields: true + minavailable: + description: The minimal available pods to run for this AppWrapper; the default value is nil + format: int32 + type: integer + priority: + description: The priority of this resource + format: int32 + type: integer + priorityslope: + description: The increasing rate of priority value for this resource + format: float + type: number + replicas: + description: Replicas is the number of desired replicas + format: int32 + type: integer + type: object + type: array + type: object + schedulingSpec: + description: SchedSpec specifies the parameters used for scheduling generic items wrapped inside AppWrappers. It defines the policy for requeuing jobs based on the number of running pods. + properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object + minAvailable: + description: Expected number of pods in running and/or completed state. Requeuing is triggered when the number of running/completed pods is not equal to this value. When not specified, requeuing is disabled and no check is performed. + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + requeuing: + description: Specification of the requeuing strategy based on waiting time. Values in this field control how often the pod check should happen, and if requeuing has reached its maximum number of times. + properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time between requeuing checks. The values available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. + type: string + initialTimeInSeconds: + description: Value to keep track of the initial wait time. Users cannot set this as it is taken from 'timeInSeconds'. + type: integer + maxNumRequeuings: + default: 0 + description: Maximum number of requeuing events allowed. Once this value is reached (e.g., 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic items are stopped and removed from the cluster (AppWrapper remains deployed). + type: integer + maxTimeInSeconds: + default: 0 + description: Maximum waiting time for requeuing checks. + type: integer + numRequeuings: + default: 0 + description: Field to keep track of how many times a requeuing event has been triggered. + type: integer + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions are checked. This value is specified by the user, but it may grow as requeuing events happen. + type: integer + type: object + type: object + selector: + description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + service: + description: AppWrapperService is App Wrapper service definition + properties: + spec: + description: ServiceSpec describes the attributes that a user creates on a service. + properties: + allocateLoadBalancerNodePorts: + description: allocateLoadBalancerNodePorts defines if NodePorts will be automatically allocated for services with type LoadBalancer. Default is "true". It may be set to "false" if the cluster load-balancer does not rely on NodePorts. If the caller requests specific NodePorts (by specifying a value), those requests will be respected, regardless of this field. This field may only be set for services with type LoadBalancer and will be cleared if the type is changed to any other type. + type: boolean + clusterIP: + description: 'clusterIP is the IP address of the service and is usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be blank) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are "None", empty string (""), or a valid IP address. Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + clusterIPs: + description: "ClusterIPs is a list of IP addresses assigned to this service, and are usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be empty) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are \"None\", empty string (\"\"), or a valid IP address. Setting this to \"None\" makes a \"headless service\" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. If this field is not specified, it will be initialized from the clusterIP field. If this field is specified, clients must ensure that clusterIPs[0] and clusterIP have the same value. \n This field may hold a maximum of two entries (dual-stack IPs, in either order). These IPs must correspond to the values of the ipFamilies field. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that discovery mechanisms will return as an alias for this service (e.g. a DNS CNAME record). No proxying will be involved. Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: externalTrafficPolicy describes how nodes distribute service traffic they receive on one of the Service's "externally-facing" addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure the service in a way that assumes that external load balancers will take care of balancing the service traffic between nodes, and so each node will deliver traffic only to the node-local endpoints of the service, without masquerading the client source IP. (Traffic mistakenly sent to a node with no endpoints will be dropped.) The default value, "Cluster", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). Note that traffic sent to an External IP or LoadBalancer IP from within the cluster will always get "Cluster" semantics, but clients sending to a NodePort from within the cluster may need to take traffic policy into account when picking a node. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck nodePort for the service. This only applies when type is set to LoadBalancer and externalTrafficPolicy is set to Local. If a value is specified, is in-range, and is not in use, it will be used. If not specified, a value will be automatically allocated. External systems (e.g. load-balancers) can use this port to determine if a given node holds endpoints for this service or not. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type). This field cannot be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: InternalTrafficPolicy describes how nodes distribute service traffic they receive on the ClusterIP. If set to "Local", the proxy will assume that pods only want to talk to endpoints of the service on the same node as the pod, dropping the traffic if there are no local endpoints. The default value, "Cluster", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). + type: string + ipFamilies: + description: "IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this service. This field is usually assigned automatically based on cluster configuration and the ipFamilyPolicy field. If this field is specified manually, the requested family is available in the cluster, and ipFamilyPolicy allows it, it will be used; otherwise creation of the service will fail. This field is conditionally mutable: it allows for adding or removing a secondary IP family, but it does not allow changing the primary IP family of the Service. Valid values are \"IPv4\" and \"IPv6\". This field only applies to Services of types ClusterIP, NodePort, and LoadBalancer, and does apply to \"headless\" services. This field will be wiped when updating a Service to type ExternalName. \n This field may hold a maximum of two entries (dual-stack families, in either order). These families must correspond to the values of the clusterIPs field, if specified. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field." + items: + description: IPFamily represents the IP Family (IPv4 or IPv6). This type is used to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: IPFamilyPolicy represents the dual-stack-ness requested or required by this Service. If there is no value provided, then this field will be set to SingleStack. Services can be "SingleStack" (a single IP family), "PreferDualStack" (two IP families on dual-stack configured clusters or a single IP family on single-stack clusters), or "RequireDualStack" (two IP families on dual-stack configured clusters, otherwise fail). The ipFamilies and clusterIPs fields depend on the value of this field. This field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: loadBalancerClass is the class of the load balancer implementation this Service belongs to. If specified, the value of this field must be a label-style identifier, with an optional prefix, e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load balancer implementation is used, today this is typically done through the cloud provider integration, but should apply for any default implementation. If set, it is assumed that a load balancer implementation is watching for Services with a matching class. Any default load balancer implementation (e.g. cloud providers) should ignore Services that set this field. This field can only be set when creating or updating a Service to type 'LoadBalancer'. Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature. Deprecated: This field was under-specified and its meaning varies across implementations, and it cannot support dual-stack. As of Kubernetes v1.24, users are encouraged to use implementation-specific annotations when available. This field may be removed in a future API version.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's port. + properties: + appProtocol: + description: The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and https://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. + type: string + name: + description: The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this service is exposed when type is NodePort or LoadBalancer. Usually assigned by the system. If a value is specified, in-range, and not in use it will be used, otherwise the operation will fail. If not specified, a port will be allocated if this Service requires one. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this service. + format: int32 + type: integer + protocol: + default: TCP + description: The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod''s container ports. If this is not specified, the value of the ''port'' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: publishNotReadyAddresses indicates that any agent which deals with endpoints for this Service should disregard any indications of ready/not-ready. The primary use case for setting this field is for a StatefulSet's Headless Service to propagate SRV DNS records for its Pods for the purpose of peer discovery. The Kubernetes controllers that generate Endpoints and EndpointSlice resources for Services interpret this to mean that all endpoints are considered "ready" even if the Pods themselves are not. Agents which consume only Kubernetes generated endpoints through the Endpoints or EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: 'type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. "ClusterIP" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object or EndpointSlice objects. If clusterIP is "None", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a virtual IP. "NodePort" builds on ClusterIP and allocates a port on every node which routes to the same endpoints as the clusterIP. "LoadBalancer" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the same endpoints as the clusterIP. "ExternalName" aliases this service to the specified externalName. Several other fields do not apply to ExternalName services. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + required: + - spec + type: object + required: + - resources + type: object + status: + description: AppWrapperStatus represents the current state of a AppWrapper + properties: + Succeeded: + description: The number of resources which reached phase Succeeded. + format: int32 + type: integer + canrun: + description: Can run? + type: boolean + conditions: + description: Represents the latest available observations of the AppWrapper's current condition. + items: + description: AppWrapperCondition describes the state of an AppWrapper at a certain point. + properties: + lastTransitionMicroTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + lastUpdateMicroTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of AppWrapper condition. + type: string + required: + - status + - type + type: object + type: array + controllerfirstdispatchtimestamp: + description: Microsecond level timestamp when controller first dispatches the AppWrapper + format: date-time + type: string + controllerfirsttimestamp: + description: Microsecond level timestamp when controller first sees QueueJob (by Informer) + format: date-time + type: string + failed: + description: The number of resources which reached phase Failed. + format: int32 + type: integer + filterignore: + description: Tell Informer to ignore this update message (do not generate a controller event) + type: boolean + isdispatched: + description: Is Dispatched? + type: boolean + local: + description: Indicate if message is a duplicate (for Informer to recognize duplicate messages) + type: boolean + message: + type: string + numberOfRequeueings: + description: Field to keep track of how many times a requeuing event has been triggered + type: integer + pending: + description: The number of pending pods. + format: int32 + type: integer + pendingpodconditions: + description: Represents the latest available observations of pods belonging to the AppWrapper. + items: + properties: + conditions: + items: + description: PodCondition contains details for the current condition of this pod. + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: 'Status is the status of the condition. Can be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + type: + description: 'Type is the type of the condition. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + required: + - status + - type + type: object + type: array + podname: + type: string + type: object + type: array + queuejobstate: + description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, ... + type: string + requeueingTimeInSeconds: + description: Field to keep track of total number of seconds spent in requeueing + type: integer + running: + format: int32 + type: integer + sender: + description: Indicate sender of this message (extremely useful for debugging) + type: string + state: + description: State - Pending, Running, Failed, Deleted + type: string + systempriority: + description: System defined Priority + format: float + type: number + template: + description: The minimal available resources to run for this AppWrapper (is this different from the MinAvailable from JobStatus) + format: int32 + type: integer + totalcpu: + description: The number of CPU consumed by all pods belonging to the AppWrapper. + format: int32 + type: integer + totalgpu: + description: The total number of GPUs consumed by all pods belonging to the AppWrapper. + format: int32 + type: integer + totalmemory: + description: The amount of memory consumed by all pods belonging to the AppWrapper. + format: int32 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/crd-quotasubtree.yml b/config/crd/crd-quotasubtree.yml new file mode 100644 index 00000000..924aaacd --- /dev/null +++ b/config/crd/crd-quotasubtree.yml @@ -0,0 +1,115 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: quotasubtrees.quota.codeflare.dev +spec: + group: quota.codeflare.dev + names: + kind: QuotaSubtree + listKind: QuotaSubtreeList + plural: quotasubtrees + singular: quotasubtree + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: QuotaSubtree is a specification for a quota subtree resource + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: QuotaSubtreeSpec is the spec for a resource plan + properties: + children: + items: + description: Child is the spec for a QuotaSubtree resource + properties: + name: + type: string + namespace: + type: string + path: + type: string + quotas: + description: Quota is the spec for a QuotaSubtree resource + properties: + disabled: + type: boolean + hardLimit: + type: boolean + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: array + parent: + type: string + parentNamespace: + type: string + type: object + status: + description: QuotaSubtreeStatus is the status for a QuotaSubtree resource + properties: + children: + items: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + type: array + totalAllocation: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + required: + - children + - totalAllocation + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/config/crd/crd-schedulingspec.yml b/config/crd/crd-schedulingspec.yml new file mode 100644 index 00000000..373dc89d --- /dev/null +++ b/config/crd/crd-schedulingspec.yml @@ -0,0 +1,80 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: schedulingspecs.workload.codeflare.dev +spec: + group: workload.codeflare.dev + names: + kind: SchedulingSpec + listKind: SchedulingSpecList + plural: schedulingspecs + singular: schedulingspec + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object + minAvailable: + description: Expected number of pods in running and/or completed state. Requeuing is triggered when the number of running/completed pods is not equal to this value. When not specified, requeuing is disabled and no check is performed. + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + requeuing: + description: Specification of the requeuing strategy based on waiting time. Values in this field control how often the pod check should happen, and if requeuing has reached its maximum number of times. + properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time between requeuing checks. The values available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. + type: string + initialTimeInSeconds: + description: Value to keep track of the initial wait time. Users cannot set this as it is taken from 'timeInSeconds'. + type: integer + maxNumRequeuings: + default: 0 + description: Maximum number of requeuing events allowed. Once this value is reached (e.g., 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic items are stopped and removed from the cluster (AppWrapper remains deployed). + type: integer + maxTimeInSeconds: + default: 0 + description: Maximum waiting time for requeuing checks. + type: integer + numRequeuings: + default: 0 + description: Field to keep track of how many times a requeuing event has been triggered. + type: integer + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions are checked. This value is specified by the user, but it may grow as requeuing events happen. + type: integer + type: object + type: object + required: + - metadata + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 73a57623..faaff445 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,8 +2,8 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- quota.codeflare_quotasubtrees.yaml -- workload.codeflare_appwrappers.yaml -- workload.codeflare_schedulingspecs.yaml +- crd-appwrapper.yml +- crd-quotasubtree.yml +- crd-schedulingspec.yml #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index e016d77f..e53d2dd9 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.36.0 +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.37.1 diff --git a/config/crd/quota.codeflare_quotasubtrees.yaml b/config/crd/quota.codeflare_quotasubtrees.yaml deleted file mode 100644 index 44bb5e0a..00000000 --- a/config/crd/quota.codeflare_quotasubtrees.yaml +++ /dev/null @@ -1,122 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: quotasubtrees.quota.codeflare.dev -spec: - group: quota.codeflare.dev - names: - kind: QuotaSubtree - listKind: QuotaSubtreeList - plural: quotasubtrees - singular: quotasubtree - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: QuotaSubtree is a specification for a quota subtree resource - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: QuotaSubtreeSpec is the spec for a resource plan - properties: - children: - items: - description: Child is the spec for a QuotaSubtree resource - properties: - name: - type: string - namespace: - type: string - path: - type: string - quotas: - description: Quota is the spec for a QuotaSubtree resource - properties: - disabled: - type: boolean - hardLimit: - type: boolean - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - type: object - type: array - parent: - type: string - parentNamespace: - type: string - type: object - status: - description: QuotaSubtreeStatus is the status for a QuotaSubtree resource - properties: - children: - items: - description: ResourceAllocation is the spec for the child status - properties: - allocated: - description: ResourceAllocationStatus is the spec for the child - resource usage - properties: - requests: - additionalProperties: - type: string - type: object - type: object - name: - type: string - namespace: - type: string - path: - type: string - type: object - type: array - totalAllocation: - description: ResourceAllocation is the spec for the child status - properties: - allocated: - description: ResourceAllocationStatus is the spec for the child - resource usage - properties: - requests: - additionalProperties: - type: string - type: object - type: object - name: - type: string - namespace: - type: string - path: - type: string - type: object - required: - - children - - totalAllocation - type: object - required: - - spec - type: object - served: true - storage: true ---- diff --git a/config/crd/workload.codeflare_appwrappers.yaml b/config/crd/workload.codeflare_appwrappers.yaml deleted file mode 100644 index e1d26041..00000000 --- a/config/crd/workload.codeflare_appwrappers.yaml +++ /dev/null @@ -1,765 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: appwrappers.workload.codeflare.dev -spec: - group: workload.codeflare.dev - names: - kind: AppWrapper - listKind: AppWrapperList - plural: appwrappers - singular: appwrapper - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: Definition of AppWrapper class - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: AppWrapperSpec describes how the App Wrapper will look like. - properties: - priority: - format: int32 - type: integer - priorityslope: - format: float - type: number - resources: - description: a collection of AppWrapperResource - properties: - GenericItems: - items: - description: AppWrapperGenericResource is App Wrapper aggregation - resource - properties: - allocated: - description: The number of allocated replicas from this - resource type - format: int32 - type: integer - completionstatus: - description: 'Optional field that drives completion status - of this AppWrapper. This field within an item of an AppWrapper - determines the full state of the AppWrapper. The completionstatus - field contains a list of conditions that make the associate - item considered completed, for instance: - completion - conditions could be "Complete" or "Failed". The associated - item''s level .status.conditions[].type field is monitored - for any one of these conditions. Once all items with this - option is set and the conditionstatus is met the entire - AppWrapper state will be changed to one of the valid AppWrapper - completion state. Note: - this is an AND operation for - all items where this option is set. See the list of AppWrapper - states for a list of valid complete states.' - type: string - custompodresources: - description: Optional section that specifies resource requirements - for non-standard k8s resources, follows same format as - that of standard k8s resources. - items: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: ResourceList is a set of (resource name, - quantity) pairs. - type: object - replicas: - type: integer - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'todo: replace with Containers []Container - Contain v1.ResourceRequirements' - type: object - required: - - replicas - - requests - type: object - type: array - generictemplate: - description: The template for the resource; it is now a - raw text because we don't know for what resource it should - be instantiated - type: object - x-kubernetes-embedded-resource: true - x-kubernetes-preserve-unknown-fields: true - minavailable: - description: The minimal available pods to run for this - AppWrapper; the default value is nil - format: int32 - type: integer - priority: - description: The priority of this resource - format: int32 - type: integer - priorityslope: - description: The increasing rate of priority value for this - resource - format: float - type: number - replicas: - description: Replicas is the number of desired replicas - format: int32 - type: integer - type: object - type: array - type: object - schedulingSpec: - description: SchedSpec specifies the parameters used for scheduling - generic items wrapped inside AppWrappers. It defines the policy - for requeuing jobs based on the number of running pods. - properties: - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds. - properties: - expected: - type: integer - limit: - type: integer - overrun: - type: boolean - type: object - minAvailable: - description: Expected number of pods in running and/or completed - state. Requeuing is triggered when the number of running/completed - pods is not equal to this value. When not specified, requeuing - is disabled and no check is performed. - type: integer - nodeSelector: - additionalProperties: - type: string - type: object - requeuing: - description: Specification of the requeuing strategy based on - waiting time. Values in this field control how often the pod - check should happen, and if requeuing has reached its maximum - number of times. - properties: - growthType: - default: exponential - description: Growth strategy to increase the waiting time - between requeuing checks. The values available are 'exponential', - 'linear', or 'none'. For example, 'exponential' growth would - double the 'timeInSeconds' value every time a requeuing - event is triggered. If the string value is misspelled or - not one of the possible options, the growth behavior is - defaulted to 'none'. - type: string - initialTimeInSeconds: - description: Value to keep track of the initial wait time. - Users cannot set this as it is taken from 'timeInSeconds'. - type: integer - maxNumRequeuings: - default: 0 - description: Maximum number of requeuing events allowed. Once - this value is reached (e.g., 'numRequeuings = maxNumRequeuings', - no more requeuing checks are performed and the generic items - are stopped and removed from the cluster (AppWrapper remains - deployed). - type: integer - maxTimeInSeconds: - default: 0 - description: Maximum waiting time for requeuing checks. - type: integer - numRequeuings: - default: 0 - description: Field to keep track of how many times a requeuing - event has been triggered. - type: integer - timeInSeconds: - default: 300 - description: Initial waiting time before requeuing conditions - are checked. This value is specified by the user, but it - may grow as requeuing events happen. - type: integer - type: object - type: object - selector: - description: A label selector is a label query over a set of resources. - The result of matchLabels and matchExpressions are ANDed. An empty - label selector matches all objects. A null label selector matches - no objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - service: - description: AppWrapperService is App Wrapper service definition - properties: - spec: - description: ServiceSpec describes the attributes that a user - creates on a service. - properties: - allocateLoadBalancerNodePorts: - description: allocateLoadBalancerNodePorts defines if NodePorts - will be automatically allocated for services with type LoadBalancer. Default - is "true". It may be set to "false" if the cluster load-balancer - does not rely on NodePorts. If the caller requests specific - NodePorts (by specifying a value), those requests will be - respected, regardless of this field. This field may only - be set for services with type LoadBalancer and will be cleared - if the type is changed to any other type. - type: boolean - clusterIP: - description: 'clusterIP is the IP address of the service and - is usually assigned randomly. If an address is specified - manually, is in-range (as per system configuration), and - is not in use, it will be allocated to the service; otherwise - creation of the service will fail. This field may not be - changed through updates unless the type field is also being - changed to ExternalName (which requires this field to be - blank) or the type field is being changed from ExternalName - (in which case this field may optionally be specified, as - describe above). Valid values are "None", empty string - (""), or a valid IP address. Setting this to "None" makes - a "headless service" (no virtual IP), which is useful when - direct endpoint connections are preferred and proxying is - not required. Only applies to types ClusterIP, NodePort, - and LoadBalancer. If this field is specified when creating - a Service of type ExternalName, creation will fail. This - field will be wiped when updating a Service to type ExternalName. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - type: string - clusterIPs: - description: "ClusterIPs is a list of IP addresses assigned - to this service, and are usually assigned randomly. If - an address is specified manually, is in-range (as per system - configuration), and is not in use, it will be allocated - to the service; otherwise creation of the service will fail. - This field may not be changed through updates unless the - type field is also being changed to ExternalName (which - requires this field to be empty) or the type field is being - changed from ExternalName (in which case this field may - optionally be specified, as describe above). Valid values - are \"None\", empty string (\"\"), or a valid IP address. - \ Setting this to \"None\" makes a \"headless service\" - (no virtual IP), which is useful when direct endpoint connections - are preferred and proxying is not required. Only applies - to types ClusterIP, NodePort, and LoadBalancer. If this - field is specified when creating a Service of type ExternalName, - creation will fail. This field will be wiped when updating - a Service to type ExternalName. If this field is not specified, - it will be initialized from the clusterIP field. If this - field is specified, clients must ensure that clusterIPs[0] - and clusterIP have the same value. \n This field may hold - a maximum of two entries (dual-stack IPs, in either order). - These IPs must correspond to the values of the ipFamilies - field. Both clusterIPs and ipFamilies are governed by the - ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" - items: - type: string - type: array - x-kubernetes-list-type: atomic - externalIPs: - description: externalIPs is a list of IP addresses for which - nodes in the cluster will also accept traffic for this service. These - IPs are not managed by Kubernetes. The user is responsible - for ensuring that traffic arrives at a node with this IP. A - common example is external load-balancers that are not part - of the Kubernetes system. - items: - type: string - type: array - externalName: - description: externalName is the external reference that discovery - mechanisms will return as an alias for this service (e.g. - a DNS CNAME record). No proxying will be involved. Must - be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) - and requires `type` to be "ExternalName". - type: string - externalTrafficPolicy: - description: externalTrafficPolicy describes how nodes distribute - service traffic they receive on one of the Service's "externally-facing" - addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). - If set to "Local", the proxy will configure the service - in a way that assumes that external load balancers will - take care of balancing the service traffic between nodes, - and so each node will deliver traffic only to the node-local - endpoints of the service, without masquerading the client - source IP. (Traffic mistakenly sent to a node with no endpoints - will be dropped.) The default value, "Cluster", uses the - standard behavior of routing to all endpoints evenly (possibly - modified by topology and other features). Note that traffic - sent to an External IP or LoadBalancer IP from within the - cluster will always get "Cluster" semantics, but clients - sending to a NodePort from within the cluster may need to - take traffic policy into account when picking a node. - type: string - healthCheckNodePort: - description: healthCheckNodePort specifies the healthcheck - nodePort for the service. This only applies when type is - set to LoadBalancer and externalTrafficPolicy is set to - Local. If a value is specified, is in-range, and is not - in use, it will be used. If not specified, a value will - be automatically allocated. External systems (e.g. load-balancers) - can use this port to determine if a given node holds endpoints - for this service or not. If this field is specified when - creating a Service which does not need it, creation will - fail. This field will be wiped when updating a Service to - no longer need it (e.g. changing type). This field cannot - be updated once set. - format: int32 - type: integer - internalTrafficPolicy: - description: InternalTrafficPolicy describes how nodes distribute - service traffic they receive on the ClusterIP. If set to - "Local", the proxy will assume that pods only want to talk - to endpoints of the service on the same node as the pod, - dropping the traffic if there are no local endpoints. The - default value, "Cluster", uses the standard behavior of - routing to all endpoints evenly (possibly modified by topology - and other features). - type: string - ipFamilies: - description: "IPFamilies is a list of IP families (e.g. IPv4, - IPv6) assigned to this service. This field is usually assigned - automatically based on cluster configuration and the ipFamilyPolicy - field. If this field is specified manually, the requested - family is available in the cluster, and ipFamilyPolicy allows - it, it will be used; otherwise creation of the service will - fail. This field is conditionally mutable: it allows for - adding or removing a secondary IP family, but it does not - allow changing the primary IP family of the Service. Valid - values are \"IPv4\" and \"IPv6\". This field only applies - to Services of types ClusterIP, NodePort, and LoadBalancer, - and does apply to \"headless\" services. This field will - be wiped when updating a Service to type ExternalName. \n - This field may hold a maximum of two entries (dual-stack - families, in either order). These families must correspond - to the values of the clusterIPs field, if specified. Both - clusterIPs and ipFamilies are governed by the ipFamilyPolicy - field." - items: - description: IPFamily represents the IP Family (IPv4 or - IPv6). This type is used to express the family of an IP - expressed by a type (e.g. service.spec.ipFamilies). - type: string - type: array - x-kubernetes-list-type: atomic - ipFamilyPolicy: - description: IPFamilyPolicy represents the dual-stack-ness - requested or required by this Service. If there is no value - provided, then this field will be set to SingleStack. Services - can be "SingleStack" (a single IP family), "PreferDualStack" - (two IP families on dual-stack configured clusters or a - single IP family on single-stack clusters), or "RequireDualStack" - (two IP families on dual-stack configured clusters, otherwise - fail). The ipFamilies and clusterIPs fields depend on the - value of this field. This field will be wiped when updating - a service to type ExternalName. - type: string - loadBalancerClass: - description: loadBalancerClass is the class of the load balancer - implementation this Service belongs to. If specified, the - value of this field must be a label-style identifier, with - an optional prefix, e.g. "internal-vip" or "example.com/internal-vip". - Unprefixed names are reserved for end-users. This field - can only be set when the Service type is 'LoadBalancer'. - If not set, the default load balancer implementation is - used, today this is typically done through the cloud provider - integration, but should apply for any default implementation. - If set, it is assumed that a load balancer implementation - is watching for Services with a matching class. Any default - load balancer implementation (e.g. cloud providers) should - ignore Services that set this field. This field can only - be set when creating or updating a Service to type 'LoadBalancer'. - Once set, it can not be changed. This field will be wiped - when a service is updated to a non 'LoadBalancer' type. - type: string - loadBalancerIP: - description: 'Only applies to Service Type: LoadBalancer. - This feature depends on whether the underlying cloud-provider - supports specifying the loadBalancerIP when a load balancer - is created. This field will be ignored if the cloud-provider - does not support the feature. Deprecated: This field was - under-specified and its meaning varies across implementations, - and it cannot support dual-stack. As of Kubernetes v1.24, - users are encouraged to use implementation-specific annotations - when available. This field may be removed in a future API - version.' - type: string - loadBalancerSourceRanges: - description: 'If specified and supported by the platform, - this will restrict traffic through the cloud-provider load-balancer - will be restricted to the specified client IPs. This field - will be ignored if the cloud-provider does not support the - feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' - items: - type: string - type: array - ports: - description: 'The list of ports that are exposed by this service. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - items: - description: ServicePort contains information on service's - port. - properties: - appProtocol: - description: The application protocol for this port. - This field follows standard Kubernetes label syntax. - Un-prefixed names are reserved for IANA standard service - names (as per RFC-6335 and https://www.iana.org/assignments/service-names). - Non-standard protocols should use prefixed names such - as mycompany.com/my-custom-protocol. - type: string - name: - description: The name of this port within the service. - This must be a DNS_LABEL. All ports within a ServiceSpec - must have unique names. When considering the endpoints - for a Service, this must match the 'name' field in - the EndpointPort. Optional if only one ServicePort - is defined on this service. - type: string - nodePort: - description: 'The port on each node on which this service - is exposed when type is NodePort or LoadBalancer. Usually - assigned by the system. If a value is specified, in-range, - and not in use it will be used, otherwise the operation - will fail. If not specified, a port will be allocated - if this Service requires one. If this field is specified - when creating a Service which does not need it, creation - will fail. This field will be wiped when updating - a Service to no longer need it (e.g. changing type - from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' - format: int32 - type: integer - port: - description: The port that will be exposed by this service. - format: int32 - type: integer - protocol: - default: TCP - description: The IP protocol for this port. Supports - "TCP", "UDP", and "SCTP". Default is TCP. - type: string - targetPort: - anyOf: - - type: integer - - type: string - description: 'Number or name of the port to access on - the pods targeted by the service. Number must be in - the range 1 to 65535. Name must be an IANA_SVC_NAME. - If this is a string, it will be looked up as a named - port in the target Pod''s container ports. If this - is not specified, the value of the ''port'' field - is used (an identity map). This field is ignored for - services with clusterIP=None, and should be omitted - or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' - x-kubernetes-int-or-string: true - required: - - port - type: object - type: array - x-kubernetes-list-map-keys: - - port - - protocol - x-kubernetes-list-type: map - publishNotReadyAddresses: - description: publishNotReadyAddresses indicates that any agent - which deals with endpoints for this Service should disregard - any indications of ready/not-ready. The primary use case - for setting this field is for a StatefulSet's Headless Service - to propagate SRV DNS records for its Pods for the purpose - of peer discovery. The Kubernetes controllers that generate - Endpoints and EndpointSlice resources for Services interpret - this to mean that all endpoints are considered "ready" even - if the Pods themselves are not. Agents which consume only - Kubernetes generated endpoints through the Endpoints or - EndpointSlice resources can safely assume this behavior. - type: boolean - selector: - additionalProperties: - type: string - description: 'Route service traffic to pods with label keys - and values matching this selector. If empty or not present, - the service is assumed to have an external process managing - its endpoints, which Kubernetes will not modify. Only applies - to types ClusterIP, NodePort, and LoadBalancer. Ignored - if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' - type: object - x-kubernetes-map-type: atomic - sessionAffinity: - description: 'Supports "ClientIP" and "None". Used to maintain - session affinity. Enable client IP based session affinity. - Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - type: string - sessionAffinityConfig: - description: sessionAffinityConfig contains the configurations - of session affinity. - properties: - clientIP: - description: clientIP contains the configurations of Client - IP based session affinity. - properties: - timeoutSeconds: - description: timeoutSeconds specifies the seconds - of ClientIP type session sticky time. The value - must be >0 && <=86400(for 1 day) if ServiceAffinity - == "ClientIP". Default value is 10800(for 3 hours). - format: int32 - type: integer - type: object - type: object - type: - description: 'type determines how the Service is exposed. - Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, - NodePort, and LoadBalancer. "ClusterIP" allocates a cluster-internal - IP address for load-balancing to endpoints. Endpoints are - determined by the selector or if that is not specified, - by manual construction of an Endpoints object or EndpointSlice - objects. If clusterIP is "None", no virtual IP is allocated - and the endpoints are published as a set of endpoints rather - than a virtual IP. "NodePort" builds on ClusterIP and allocates - a port on every node which routes to the same endpoints - as the clusterIP. "LoadBalancer" builds on NodePort and - creates an external load-balancer (if supported in the current - cloud) which routes to the same endpoints as the clusterIP. - "ExternalName" aliases this service to the specified externalName. - Several other fields do not apply to ExternalName services. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' - type: string - type: object - required: - - spec - type: object - required: - - resources - type: object - status: - description: AppWrapperStatus represents the current state of a AppWrapper - properties: - Succeeded: - description: The number of resources which reached phase Succeeded. - format: int32 - type: integer - canrun: - description: Can run? - type: boolean - conditions: - description: Represents the latest available observations of the AppWrapper's - current condition. - items: - description: AppWrapperCondition describes the state of an AppWrapper - at a certain point. - properties: - lastTransitionMicroTime: - description: Last time the condition transitioned from one status - to another. - format: date-time - type: string - lastUpdateMicroTime: - description: The last time this condition was updated. - format: date-time - type: string - message: - description: A human-readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of AppWrapper condition. - type: string - required: - - status - - type - type: object - type: array - controllerfirstdispatchtimestamp: - description: Microsecond level timestamp when controller first dispatches - the AppWrapper - format: date-time - type: string - controllerfirsttimestamp: - description: Microsecond level timestamp when controller first sees - QueueJob (by Informer) - format: date-time - type: string - failed: - description: The number of resources which reached phase Failed. - format: int32 - type: integer - filterignore: - description: Tell Informer to ignore this update message (do not generate - a controller event) - type: boolean - isdispatched: - description: Is Dispatched? - type: boolean - local: - description: Indicate if message is a duplicate (for Informer to recognize - duplicate messages) - type: boolean - message: - type: string - numberOfRequeueings: - description: Field to keep track of how many times a requeuing event - has been triggered - type: integer - pending: - description: The number of pending pods. - format: int32 - type: integer - pendingpodconditions: - description: Represents the latest available observations of pods - belonging to the AppWrapper. - items: - properties: - conditions: - items: - description: PodCondition contains details for the current - condition of this pod. - properties: - lastProbeTime: - description: Last time we probed the condition. - format: date-time - type: string - lastTransitionTime: - description: Last time the condition transitioned from - one status to another. - format: date-time - type: string - message: - description: Human-readable message indicating details - about last transition. - type: string - reason: - description: Unique, one-word, CamelCase reason for the - condition's last transition. - type: string - status: - description: 'Status is the status of the condition. Can - be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' - type: string - type: - description: 'Type is the type of the condition. More - info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' - type: string - required: - - status - - type - type: object - type: array - podname: - type: string - type: object - type: array - queuejobstate: - description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, - ... - type: string - requeueingTimeInSeconds: - description: Field to keep track of total number of seconds spent - in requeueing - type: integer - running: - format: int32 - type: integer - sender: - description: Indicate sender of this message (extremely useful for - debugging) - type: string - state: - description: State - Pending, Running, Failed, Deleted - type: string - systempriority: - description: System defined Priority - format: float - type: number - template: - description: The minimal available resources to run for this AppWrapper - (is this different from the MinAvailable from JobStatus) - format: int32 - type: integer - totalcpu: - description: The number of CPU consumed by all pods belonging to the - AppWrapper. - format: int32 - type: integer - totalgpu: - description: The total number of GPUs consumed by all pods belonging - to the AppWrapper. - format: int32 - type: integer - totalmemory: - description: The amount of memory consumed by all pods belonging to - the AppWrapper. - format: int32 - type: integer - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} ---- diff --git a/config/crd/workload.codeflare_schedulingspecs.yaml b/config/crd/workload.codeflare_schedulingspecs.yaml deleted file mode 100644 index fc44a432..00000000 --- a/config/crd/workload.codeflare_schedulingspecs.yaml +++ /dev/null @@ -1,102 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: schedulingspecs.workload.codeflare.dev -spec: - group: workload.codeflare.dev - names: - kind: SchedulingSpec - listKind: SchedulingSpecList - plural: schedulingspecs - singular: schedulingspec - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds. - properties: - expected: - type: integer - limit: - type: integer - overrun: - type: boolean - type: object - minAvailable: - description: Expected number of pods in running and/or completed state. - Requeuing is triggered when the number of running/completed pods - is not equal to this value. When not specified, requeuing is disabled - and no check is performed. - type: integer - nodeSelector: - additionalProperties: - type: string - type: object - requeuing: - description: Specification of the requeuing strategy based on waiting - time. Values in this field control how often the pod check should - happen, and if requeuing has reached its maximum number of times. - properties: - growthType: - default: exponential - description: Growth strategy to increase the waiting time between - requeuing checks. The values available are 'exponential', 'linear', - or 'none'. For example, 'exponential' growth would double the - 'timeInSeconds' value every time a requeuing event is triggered. - If the string value is misspelled or not one of the possible - options, the growth behavior is defaulted to 'none'. - type: string - initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users - cannot set this as it is taken from 'timeInSeconds'. - type: integer - maxNumRequeuings: - default: 0 - description: Maximum number of requeuing events allowed. Once - this value is reached (e.g., 'numRequeuings = maxNumRequeuings', - no more requeuing checks are performed and the generic items - are stopped and removed from the cluster (AppWrapper remains - deployed). - type: integer - maxTimeInSeconds: - default: 0 - description: Maximum waiting time for requeuing checks. - type: integer - numRequeuings: - default: 0 - description: Field to keep track of how many times a requeuing - event has been triggered. - type: integer - timeInSeconds: - default: 300 - description: Initial waiting time before requeuing conditions - are checked. This value is specified by the user, but it may - grow as requeuing events happen. - type: integer - type: object - type: object - required: - - metadata - type: object - served: true - storage: true From b051b1a64a0dffcd0c0195758152c5f43a7908ff Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Tue, 24 Oct 2023 11:06:50 +0100 Subject: [PATCH 207/377] addressing feedback --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d89caf1c..2e6ab468 100644 --- a/Makefile +++ b/Makefile @@ -151,8 +151,7 @@ defaults: manifests: controller-gen kustomize install-yq ## Generate RBAC objects. $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml - $(KUSTOMIZE) build config/crd/mcad > config/crd/mcad.yaml - $(YQ) -s '"crd-" + .spec.names.singular' config/crd/mcad.yaml --no-doc + $(KUSTOMIZE) build config/crd/mcad | $(YQ) -s '"crd-" + .spec.names.singular' --no-doc mv crd-*.yml config/crd .PHONY: fmt From 7e6da4cc2f3848546bc7bd1893bc9da6df90feba Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 24 Oct 2023 10:50:27 +0100 Subject: [PATCH 208/377] Updated CFO sync and tag-build workflows --- .github/workflows/odh-fork-sync.yml | 9 ++++++--- .github/workflows/tag-and-build.yml | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/odh-fork-sync.yml b/.github/workflows/odh-fork-sync.yml index 72d4a366..690a5937 100644 --- a/.github/workflows/odh-fork-sync.yml +++ b/.github/workflows/odh-fork-sync.yml @@ -1,7 +1,8 @@ name: Call sync on OpenDataHub CFO fork sync on: release: - types: [published] + types: + [published] workflow_dispatch: jobs: @@ -9,7 +10,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Sync-fork - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} run: | gh workflow run sync-fork.yaml -R github.com/opendatahub-io/codeflare-operator -r main + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: + bash diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 3fa7484d..2ecdfb72 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -171,5 +171,5 @@ jobs: gh release edit ${{ github.event.inputs.version }} --notes-file release-notes.md rm release-notes.md env: - GITHUB_TOKEN: ${{ github.TOKEN }} + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash From 6d3d1221c2027e58d39c4515327d0b7141f3f756 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 24 Oct 2023 11:00:33 +0100 Subject: [PATCH 209/377] Fixed formatting --- .github/workflows/odh-fork-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/odh-fork-sync.yml b/.github/workflows/odh-fork-sync.yml index 690a5937..3d8b3949 100644 --- a/.github/workflows/odh-fork-sync.yml +++ b/.github/workflows/odh-fork-sync.yml @@ -1,7 +1,7 @@ name: Call sync on OpenDataHub CFO fork sync on: release: - types: + types: [published] workflow_dispatch: From 349d05009ebe9270b7b92c66b8a0d809c8a7e7a6 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 24 Oct 2023 11:26:53 +0100 Subject: [PATCH 210/377] Updated release types --- .github/workflows/odh-fork-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/odh-fork-sync.yml b/.github/workflows/odh-fork-sync.yml index 3d8b3949..7c5a393d 100644 --- a/.github/workflows/odh-fork-sync.yml +++ b/.github/workflows/odh-fork-sync.yml @@ -2,7 +2,7 @@ name: Call sync on OpenDataHub CFO fork sync on: release: types: - [published] + - published workflow_dispatch: jobs: From 47bd7064e90cd387bbb22e34708c20139324b8f9 Mon Sep 17 00:00:00 2001 From: Mark Campbell Date: Tue, 24 Oct 2023 12:15:00 +0100 Subject: [PATCH 211/377] Update .github/workflows/odh-fork-sync.yml Co-authored-by: Karel Suta --- .github/workflows/odh-fork-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/odh-fork-sync.yml b/.github/workflows/odh-fork-sync.yml index 7c5a393d..22f42f00 100644 --- a/.github/workflows/odh-fork-sync.yml +++ b/.github/workflows/odh-fork-sync.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Sync-fork run: | - gh workflow run sync-fork.yaml -R github.com/opendatahub-io/codeflare-operator -r main + gh workflow run sync-fork.yaml --repo github.com/opendatahub-io/codeflare-operator --ref main env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: From 14c6fb4e8608772049ea2f19ce498e9e3abe359c Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 13 Oct 2023 15:48:14 +0100 Subject: [PATCH 212/377] removing support package --- test/support/batch.go | 37 -------- test/support/conditions.go | 85 ----------------- test/support/core.go | 112 ---------------------- test/support/defaults.go | 11 --- test/support/events.go | 148 ----------------------------- test/support/ingress.go | 101 -------------------- test/support/mcad.go | 56 ----------- test/support/namespace.go | 55 ----------- test/support/ocm.go | 74 --------------- test/support/openshift.go | 33 ------- test/support/ray_api.go | 45 --------- test/support/ray_cluster_client.go | 128 ------------------------- test/support/route.go | 91 ------------------ test/support/service.go | 27 ------ test/support/support.go | 76 --------------- test/support/test.go | 139 --------------------------- test/support/utils.go | 41 -------- 17 files changed, 1259 deletions(-) delete mode 100644 test/support/batch.go delete mode 100644 test/support/conditions.go delete mode 100644 test/support/core.go delete mode 100644 test/support/defaults.go delete mode 100644 test/support/events.go delete mode 100644 test/support/ingress.go delete mode 100644 test/support/mcad.go delete mode 100644 test/support/namespace.go delete mode 100644 test/support/ocm.go delete mode 100644 test/support/openshift.go delete mode 100644 test/support/ray_api.go delete mode 100644 test/support/ray_cluster_client.go delete mode 100644 test/support/route.go delete mode 100644 test/support/service.go delete mode 100644 test/support/support.go delete mode 100644 test/support/test.go delete mode 100644 test/support/utils.go diff --git a/test/support/batch.go b/test/support/batch.go deleted file mode 100644 index 2cb2d543..00000000 --- a/test/support/batch.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - - batchv1 "k8s.io/api/batch/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func Job(t Test, namespace, name string) func(g gomega.Gomega) *batchv1.Job { - return func(g gomega.Gomega) *batchv1.Job { - job, err := t.Client().Core().BatchV1().Jobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return job - } -} - -func GetJob(t Test, namespace, name string) *batchv1.Job { - t.T().Helper() - return Job(t, namespace, name)(t) -} diff --git a/test/support/conditions.go b/test/support/conditions.go deleted file mode 100644 index 4ef4be1e..00000000 --- a/test/support/conditions.go +++ /dev/null @@ -1,85 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - - routev1 "github.com/openshift/api/route/v1" -) - -type conditionType interface { - ~string -} - -func ConditionStatus[T conditionType](conditionType T) func(any) corev1.ConditionStatus { - return func(object any) corev1.ConditionStatus { - switch o := object.(type) { - - case *batchv1.Job: - if c := getJobCondition(o.Status.Conditions, batchv1.JobConditionType(conditionType)); c != nil { - return c.Status - } - case *appsv1.Deployment: - if c := getDeploymentCondition(o.Status.Conditions, appsv1.DeploymentConditionType(conditionType)); c != nil { - return c.Status - } - case *routev1.Route: - if len(o.Status.Ingress) == 0 { - // Route is not initialized yet - break - } - if c := getRouteCondition(o.Status.Ingress[0].Conditions, routev1.RouteIngressConditionType(conditionType)); c != nil { - return c.Status - } - } - - return corev1.ConditionUnknown - } -} - -// TODO: to be replaced with a generic version once common struct fields of a type set can be used. -// See https://github.com/golang/go/issues/48522 -func getJobCondition(conditions []batchv1.JobCondition, conditionType batchv1.JobConditionType) *batchv1.JobCondition { - for _, c := range conditions { - if c.Type == conditionType { - return &c - } - } - return nil -} - -func getDeploymentCondition(conditions []appsv1.DeploymentCondition, conditionType appsv1.DeploymentConditionType) *appsv1.DeploymentCondition { - for _, c := range conditions { - if c.Type == conditionType { - return &c - } - } - return nil -} - -func getRouteCondition(conditions []routev1.RouteIngressCondition, conditionType routev1.RouteIngressConditionType) *routev1.RouteIngressCondition { - for _, c := range conditions { - if c.Type == conditionType { - return &c - } - } - return nil -} diff --git a/test/support/core.go b/test/support/core.go deleted file mode 100644 index 70c48c20..00000000 --- a/test/support/core.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "encoding/json" - "io" - - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -func CreateConfigMap(t Test, namespace string, content map[string][]byte) *corev1.ConfigMap { - configMap := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "config-", - Namespace: namespace, - }, - BinaryData: content, - Immutable: Ptr(true), - } - - configMap, err := t.Client().Core().CoreV1().ConfigMaps(namespace).Create(t.Ctx(), configMap, metav1.CreateOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - t.T().Logf("Created ConfigMap %s/%s successfully", configMap.Namespace, configMap.Name) - - return configMap -} - -func Raw(t Test, obj runtime.Object) runtime.RawExtension { - t.T().Helper() - data, err := json.Marshal(obj) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return runtime.RawExtension{ - Raw: data, - } -} - -func GetPods(t Test, namespace string, options metav1.ListOptions) []corev1.Pod { - t.T().Helper() - pods, err := t.Client().Core().CoreV1().Pods(namespace).List(t.Ctx(), options) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return pods.Items -} - -func GetPodLogs(t Test, pod *corev1.Pod, options corev1.PodLogOptions) []byte { - t.T().Helper() - stream, err := t.Client().Core().CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetName(), &options).Stream(t.Ctx()) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - t.Expect(stream.Close()).To(gomega.Succeed()) - }() - - bytes, err := io.ReadAll(stream) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - return bytes -} - -func storeAllPodLogs(t Test, namespace *corev1.Namespace) { - t.T().Helper() - - pods, err := t.Client().Core().CoreV1().Pods(namespace.Name).List(t.Ctx(), metav1.ListOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - for _, pod := range pods.Items { - for _, container := range pod.Spec.Containers { - t.T().Logf("Retrieving Pod Container %s/%s/%s logs", pod.Namespace, pod.Name, container.Name) - storeContainerLog(t, namespace, pod.Name, container.Name) - } - } -} - -func storeContainerLog(t Test, namespace *corev1.Namespace, podName, containerName string) { - t.T().Helper() - - options := corev1.PodLogOptions{Container: containerName} - stream, err := t.Client().Core().CoreV1().Pods(namespace.Name).GetLogs(podName, &options).Stream(t.Ctx()) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - defer func() { - t.Expect(stream.Close()).To(gomega.Succeed()) - }() - - bytes, err := io.ReadAll(stream) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - containerLogFileName := "pod-" + podName + "-" + containerName - WriteToOutputDir(t, containerLogFileName, Log, bytes) -} diff --git a/test/support/defaults.go b/test/support/defaults.go deleted file mode 100644 index 2e0227cf..00000000 --- a/test/support/defaults.go +++ /dev/null @@ -1,11 +0,0 @@ -package support - -// *********************** -// DO NOT EDIT THIS FILE -// *********************** - -const ( - CodeFlareSDKVersion = "0.10.1" - RayVersion = "2.5.0" - RayImage = "rayproject/ray:2.5.0" -) diff --git a/test/support/events.go b/test/support/events.go deleted file mode 100644 index beafe558..00000000 --- a/test/support/events.go +++ /dev/null @@ -1,148 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "bytes" - "fmt" - - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - eventsv1 "k8s.io/api/events/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// Based on https://github.com/apache/incubator-kie-kogito-operator/blob/28b2d3dc945e48659b199cca33723568b848f72e/test/pkg/framework/logging.go - -const ( - eventLastSeenKey = "LAST_SEEN" - eventFirstSeenKey = "FIRST_SEEN" - eventNameKey = "NAME" - eventSubObjectKey = "SUBOBJECT" - eventTypeKey = "TYPE" - eventReasonKey = "REASON" - eventMessageKey = "MESSAGE" - - eventLogFileName = "events" -) - -var eventKeys = []string{ - eventLastSeenKey, - eventFirstSeenKey, - eventNameKey, - eventSubObjectKey, - eventTypeKey, - eventReasonKey, - eventMessageKey, -} - -func storeEvents(t Test, namespace *corev1.Namespace) { - t.T().Helper() - - events, err := t.Client().Core().EventsV1().Events(namespace.Name).List(t.Ctx(), metav1.ListOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - bytes, err := renderEventContent(eventKeys, mapEventsToKeys(events)) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - WriteToOutputDir(t, eventLogFileName, Log, bytes) -} - -func mapEventsToKeys(eventList *eventsv1.EventList) []map[string]string { - eventMaps := []map[string]string{} - - for _, event := range eventList.Items { - eventMap := make(map[string]string) - eventMap[eventLastSeenKey] = getDefaultEventValueIfNull(event.DeprecatedLastTimestamp.Format("2006-01-02 15:04:05")) - eventMap[eventFirstSeenKey] = getDefaultEventValueIfNull(event.DeprecatedFirstTimestamp.Format("2006-01-02 15:04:05")) - eventMap[eventNameKey] = getDefaultEventValueIfNull(event.GetName()) - eventMap[eventSubObjectKey] = getDefaultEventValueIfNull(event.Regarding.FieldPath) - eventMap[eventTypeKey] = getDefaultEventValueIfNull(event.Type) - eventMap[eventReasonKey] = getDefaultEventValueIfNull(event.Reason) - eventMap[eventMessageKey] = getDefaultEventValueIfNull(event.Note) - - eventMaps = append(eventMaps, eventMap) - } - return eventMaps -} - -func getDefaultEventValueIfNull(value string) string { - if len(value) <= 0 { - return "-" - } - return value -} - -func renderEventContent(keys []string, dataMaps []map[string]string) ([]byte, error) { - var content bytes.Buffer - // Get size of strings to be written, to be able to format correctly - maxStringSizeMap := make(map[string]int) - for _, key := range keys { - maxSize := len(key) - for _, dataMap := range dataMaps { - if len(dataMap[key]) > maxSize { - maxSize = len(dataMap[key]) - } - } - maxStringSizeMap[key] = maxSize - } - - // Write headers - for _, header := range keys { - if _, err := content.WriteString(header); err != nil { - return nil, fmt.Errorf("error in writing the header: %v", err) - } - if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[header] - len(header) + 1)); err != nil { - return nil, fmt.Errorf("error in writing headers: %v", err) - } - if _, err := content.WriteString(" | "); err != nil { - return nil, fmt.Errorf("error in writing headers : %v", err) - } - } - if _, err := content.WriteString("\n"); err != nil { - return nil, fmt.Errorf("error in writing headers '|': %v", err) - - } - - // Write events - for _, dataMap := range dataMaps { - for _, key := range keys { - if _, err := content.WriteString(dataMap[key]); err != nil { - return nil, fmt.Errorf("error in writing events: %v", err) - } - if _, err := content.WriteString(getWhitespaceStr(maxStringSizeMap[key] - len(dataMap[key]) + 1)); err != nil { - return nil, fmt.Errorf("error in writing events: %v", err) - } - if _, err := content.WriteString(" | "); err != nil { - return nil, fmt.Errorf("error in writing events: %v", err) - } - } - if _, err := content.WriteString("\n"); err != nil { - return nil, fmt.Errorf("error in writing events: %v", err) - } - } - return content.Bytes(), nil -} - -func getWhitespaceStr(size int) string { - whiteSpaceStr := "" - for i := 0; i < size; i++ { - whiteSpaceStr += " " - } - return whiteSpaceStr -} diff --git a/test/support/ingress.go b/test/support/ingress.go deleted file mode 100644 index d834f3b1..00000000 --- a/test/support/ingress.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "net/url" - - "github.com/onsi/gomega" - - networkingv1 "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func Ingress(t Test, namespace, name string) func(g gomega.Gomega) *networkingv1.Ingress { - return func(g gomega.Gomega) *networkingv1.Ingress { - ingress, err := t.Client().Core().NetworkingV1().Ingresses(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return ingress - } -} - -func GetIngress(t Test, namespace, name string) *networkingv1.Ingress { - t.T().Helper() - return Ingress(t, namespace, name)(t) -} - -func LoadBalancerIngresses(ingress *networkingv1.Ingress) []networkingv1.IngressLoadBalancerIngress { - return ingress.Status.LoadBalancer.Ingress -} - -func ExposeServiceByIngress(t Test, name string, namespace string, serviceName string, servicePort string) url.URL { - ingress := &networkingv1.Ingress{ - TypeMeta: metav1.TypeMeta{ - APIVersion: networkingv1.SchemeGroupVersion.String(), - Kind: "Ingress", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Annotations: map[string]string{ - "nginx.ingress.kubernetes.io/use-regex": "true", - "nginx.ingress.kubernetes.io/rewrite-target": "/$2", - }, - }, - Spec: networkingv1.IngressSpec{ - Rules: []networkingv1.IngressRule{ - networkingv1.IngressRule{ - IngressRuleValue: networkingv1.IngressRuleValue{ - HTTP: &networkingv1.HTTPIngressRuleValue{ - Paths: []networkingv1.HTTPIngressPath{ - networkingv1.HTTPIngressPath{ - Path: "/" + name + "(/|$)(.*)", - PathType: Ptr(networkingv1.PathTypePrefix), - Backend: networkingv1.IngressBackend{ - Service: &networkingv1.IngressServiceBackend{ - Name: serviceName, - Port: networkingv1.ServiceBackendPort{ - Name: servicePort, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - _, err := t.Client().Core().NetworkingV1().Ingresses(ingress.Namespace).Create(t.Ctx(), ingress, metav1.CreateOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - t.T().Logf("Created Ingress %s/%s successfully", ingress.Namespace, ingress.Name) - - t.T().Logf("Waiting for Ingress %s/%s to be admitted", ingress.Namespace, ingress.Name) - t.Eventually(Ingress(t, ingress.Namespace, ingress.Name), TestTimeoutMedium). - Should(gomega.WithTransform(LoadBalancerIngresses, gomega.HaveLen(1))) - - ingress = GetIngress(t, ingress.Namespace, ingress.Name) - - ingressURL := url.URL{ - Scheme: "http", - Host: ingress.Status.LoadBalancer.Ingress[0].IP, - Path: name, - } - return ingressURL -} diff --git a/test/support/mcad.go b/test/support/mcad.go deleted file mode 100644 index 8fdda165..00000000 --- a/test/support/mcad.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func AppWrapper(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *mcadv1beta1.AppWrapper { - return func(g gomega.Gomega) *mcadv1beta1.AppWrapper { - aw, err := t.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return aw - } -} - -func AppWrappers(t Test, namespace *corev1.Namespace) func(g gomega.Gomega) []*mcadv1beta1.AppWrapper { - return func(g gomega.Gomega) []*mcadv1beta1.AppWrapper { - aws, err := t.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).List(t.Ctx(), metav1.ListOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - - awsp := []*mcadv1beta1.AppWrapper{} - for _, v := range aws.Items { - v := v - awsp = append(awsp, &v) - } - - return awsp - } -} - -func AppWrapperName(aw *mcadv1beta1.AppWrapper) string { - return aw.Name -} - -func AppWrapperState(aw *mcadv1beta1.AppWrapper) mcadv1beta1.AppWrapperState { - return aw.Status.State -} diff --git a/test/support/namespace.go b/test/support/namespace.go deleted file mode 100644 index 145acbb4..00000000 --- a/test/support/namespace.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func createTestNamespace(t Test, options ...Option[*corev1.Namespace]) *corev1.Namespace { - t.T().Helper() - namespace := &corev1.Namespace{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "Namespace", - }, - ObjectMeta: metav1.ObjectMeta{ - GenerateName: "test-ns-", - }, - } - - for _, option := range options { - t.Expect(option.applyTo(namespace)).To(gomega.Succeed()) - } - - namespace, err := t.Client().Core().CoreV1().Namespaces().Create(t.Ctx(), namespace, metav1.CreateOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - - return namespace -} - -func deleteTestNamespace(t Test, namespace *corev1.Namespace) { - t.T().Helper() - propagationPolicy := metav1.DeletePropagationBackground - err := t.Client().Core().CoreV1().Namespaces().Delete(t.Ctx(), namespace.Name, metav1.DeleteOptions{ - PropagationPolicy: &propagationPolicy, - }) - t.Expect(err).NotTo(gomega.HaveOccurred()) -} diff --git a/test/support/ocm.go b/test/support/ocm.go deleted file mode 100644 index 3d6bd7f1..00000000 --- a/test/support/ocm.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "fmt" - "os" - - "github.com/onsi/gomega" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - ocmsdk "github.com/openshift-online/ocm-sdk-go" - cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" -) - -func CreateOCMConnection(test Test) *ocmsdk.Connection { - secretNamespace, secretName := GetInstascaleOcmSecret() - instascaleOCMSecret, err := test.Client().Core().CoreV1().Secrets(secretNamespace).Get(test.Ctx(), secretName, metav1.GetOptions{}) - test.Expect(err).NotTo(gomega.HaveOccurred()) - - ocmToken := string(instascaleOCMSecret.Data["token"]) - test.T().Logf("Retrieved Secret %s/%s successfully", instascaleOCMSecret.Namespace, instascaleOCMSecret.Name) - - connection, err := buildOCMConnection(ocmToken) - test.Expect(err).NotTo(gomega.HaveOccurred()) - return connection -} - -func buildOCMConnection(secret string) (*ocmsdk.Connection, error) { - connection, err := ocmsdk.NewConnectionBuilder(). - Tokens(secret). - Build() - if err != nil { - fmt.Fprintf(os.Stderr, "Can't build connection: %v\n", err) - return nil, err - } - - return connection, nil -} - -func MachinePools(t Test, connection *ocmsdk.Connection) func(g gomega.Gomega) []*cmv1.MachinePool { - osdClusterId, found := GetOsdClusterId() - t.Expect(found).To(gomega.BeTrue(), "OSD cluster id not found, please configure environment properly") - - return func(g gomega.Gomega) []*cmv1.MachinePool { - machinePoolsListResponse, err := connection.ClustersMgmt().V1().Clusters().Cluster(osdClusterId).MachinePools().List().Send() - g.Expect(err).NotTo(gomega.HaveOccurred()) - return machinePoolsListResponse.Items().Slice() - } -} - -func GetMachinePools(t Test, connection *ocmsdk.Connection) []*cmv1.MachinePool { - t.T().Helper() - return MachinePools(t, connection)(t) -} - -func MachinePoolId(machinePool *cmv1.MachinePool) string { - return machinePool.ID() -} diff --git a/test/support/openshift.go b/test/support/openshift.go deleted file mode 100644 index cfe3b5a3..00000000 --- a/test/support/openshift.go +++ /dev/null @@ -1,33 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - - "k8s.io/apimachinery/pkg/api/errors" -) - -func IsOpenShift(test Test) bool { - test.T().Helper() - _, err := test.Client().Core().Discovery().ServerResourcesForGroupVersion("image.openshift.io/v1") - if err != nil && errors.IsNotFound(err) { - return false - } - test.Expect(err).NotTo(gomega.HaveOccurred()) - return true -} diff --git a/test/support/ray_api.go b/test/support/ray_api.go deleted file mode 100644 index 1f04f6dd..00000000 --- a/test/support/ray_api.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" -) - -func GetRayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) *RayJobDetailsResponse { - t.T().Helper() - return RayJobAPIDetails(t, rayClient, jobID)(t) -} - -func WriteRayJobAPILogs(t Test, rayClient RayClusterClient, jobID string) { - t.T().Helper() - logs, err := rayClient.GetJobLogs(jobID) - t.Expect(err).NotTo(gomega.HaveOccurred()) - WriteToOutputDir(t, "ray-job-log-"+jobID, Log, []byte(logs)) -} - -func RayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) func(g gomega.Gomega) *RayJobDetailsResponse { - return func(g gomega.Gomega) *RayJobDetailsResponse { - jobDetails, err := rayClient.GetJobDetails(jobID) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return jobDetails - } -} - -func GetRayJobAPIDetailsStatus(jobDetails *RayJobDetailsResponse) string { - return jobDetails.Status -} diff --git a/test/support/ray_cluster_client.go b/test/support/ray_cluster_client.go deleted file mode 100644 index bbce9af2..00000000 --- a/test/support/ray_cluster_client.go +++ /dev/null @@ -1,128 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" -) - -type RayJobSetup struct { - EntryPoint string `json:"entrypoint"` - RuntimeEnv map[string]any `json:"runtime_env"` -} - -type RayJobResponse struct { - JobID string `json:"job_id"` - SubmissionID string `json:"submission_id"` -} - -type RayJobDetailsResponse struct { - JobID string `json:"job_id"` - SubmissionID string `json:"submission_id"` - Status string `json:"status"` -} - -type RayJobLogsResponse struct { - Logs string `json:"logs"` -} - -var _ RayClusterClient = (*rayClusterClient)(nil) - -type rayClusterClient struct { - endpoint url.URL -} - -type RayClusterClient interface { - CreateJob(job *RayJobSetup) (*RayJobResponse, error) - GetJobDetails(jobID string) (*RayJobDetailsResponse, error) - GetJobLogs(jobID string) (string, error) -} - -func NewRayClusterClient(dashboardEndpoint url.URL) RayClusterClient { - return &rayClusterClient{endpoint: dashboardEndpoint} -} - -func (client *rayClusterClient) CreateJob(job *RayJobSetup) (response *RayJobResponse, err error) { - marshalled, err := json.Marshal(job) - if err != nil { - return - } - - createJobURL := client.endpoint.String() + "/api/jobs/" - resp, err := http.Post(createJobURL, "application/json", bytes.NewReader(marshalled)) - if err != nil { - return - } - - respData, err := io.ReadAll(resp.Body) - if err != nil { - return - } - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("incorrect response code: %d for creating Ray Job, response body: %s", resp.StatusCode, respData) - } - - err = json.Unmarshal(respData, &response) - return -} - -func (client *rayClusterClient) GetJobDetails(jobID string) (response *RayJobDetailsResponse, err error) { - getJobDetailsURL := client.endpoint.String() + "/api/jobs/" + jobID - resp, err := http.Get(getJobDetailsURL) - if err != nil { - return - } - - respData, err := io.ReadAll(resp.Body) - if err != nil { - return - } - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("incorrect response code: %d for retrieving Ray Job details, response body: %s", resp.StatusCode, respData) - } - - err = json.Unmarshal(respData, &response) - return -} - -func (client *rayClusterClient) GetJobLogs(jobID string) (logs string, err error) { - getJobLogsURL := client.endpoint.String() + "/api/jobs/" + jobID + "/logs" - resp, err := http.Get(getJobLogsURL) - if err != nil { - return - } - - respData, err := io.ReadAll(resp.Body) - if err != nil { - return - } - - if resp.StatusCode != 200 { - return "", fmt.Errorf("incorrect response code: %d for retrieving Ray Job logs, response body: %s", resp.StatusCode, respData) - } - - jobLogs := RayJobLogsResponse{} - err = json.Unmarshal(respData, &jobLogs) - return jobLogs.Logs, err -} diff --git a/test/support/route.go b/test/support/route.go deleted file mode 100644 index df160890..00000000 --- a/test/support/route.go +++ /dev/null @@ -1,91 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "net/http" - "net/url" - - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - routev1 "github.com/openshift/api/route/v1" -) - -func Route(t Test, namespace, name string) func(g gomega.Gomega) *routev1.Route { - return func(g gomega.Gomega) *routev1.Route { - route, err := t.Client().Route().RouteV1().Routes(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return route - } -} - -func GetRoute(t Test, namespace, name string) *routev1.Route { - t.T().Helper() - return Route(t, namespace, name)(t) -} - -func ExposeServiceByRoute(t Test, name string, namespace string, serviceName string, servicePort string) url.URL { - r := &routev1.Route{ - TypeMeta: metav1.TypeMeta{ - APIVersion: routev1.SchemeGroupVersion.String(), - Kind: "Route", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: routev1.RouteSpec{ - To: routev1.RouteTargetReference{ - Name: serviceName, - }, - Port: &routev1.RoutePort{ - TargetPort: intstr.FromString(servicePort), - }, - }, - } - - _, err := t.Client().Route().RouteV1().Routes(r.Namespace).Create(t.Ctx(), r, metav1.CreateOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - t.T().Logf("Created Route %s/%s successfully", r.Namespace, r.Name) - - t.T().Logf("Waiting for Route %s/%s to be available", r.Namespace, r.Name) - t.Eventually(Route(t, r.Namespace, r.Name), TestTimeoutLong). - Should(gomega.WithTransform(ConditionStatus(routev1.RouteAdmitted), gomega.Equal(corev1.ConditionTrue))) - - // Retrieve hostname - r, err = t.Client().Route().RouteV1().Routes(r.Namespace).Get(t.Ctx(), r.Name, metav1.GetOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - hostname := r.Status.Ingress[0].Host - - // Wait for expected HTTP code - t.Eventually(func() int { - resp, _ := http.Get("http://" + hostname) - return resp.StatusCode - }, TestTimeoutLong).Should(gomega.Not(gomega.Equal(503))) - - r = GetRoute(t, r.Namespace, r.Name) - routeURL := url.URL{ - Scheme: "http", - Host: r.Status.Ingress[0].Host, - } - - return routeURL -} diff --git a/test/support/service.go b/test/support/service.go deleted file mode 100644 index d59ea3ec..00000000 --- a/test/support/service.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import "net/url" - -func ExposeService(t Test, name string, namespace string, serviceName string, servicePort string) url.URL { - if IsOpenShift(t) { - return ExposeServiceByRoute(t, name, namespace, serviceName, servicePort) - } else { - return ExposeServiceByIngress(t, name, namespace, serviceName, servicePort) - } -} diff --git a/test/support/support.go b/test/support/support.go deleted file mode 100644 index 36c8c9be..00000000 --- a/test/support/support.go +++ /dev/null @@ -1,76 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "fmt" - "os" - "time" - - "github.com/onsi/gomega" - "github.com/onsi/gomega/format" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -var ( - ApplyOptions = metav1.ApplyOptions{FieldManager: "codeflare-test", Force: true} - - TestTimeoutShort = 1 * time.Minute - TestTimeoutMedium = 2 * time.Minute - TestTimeoutLong = 5 * time.Minute - TestTimeoutGpuProvisioning = 30 * time.Minute -) - -func init() { - if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_SHORT"); ok { - if duration, err := time.ParseDuration(value); err == nil { - TestTimeoutShort = duration - } else { - fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_SHORT. Using default value: %s", TestTimeoutShort) - } - } - if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_MEDIUM"); ok { - if duration, err := time.ParseDuration(value); err == nil { - TestTimeoutMedium = duration - } else { - fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_MEDIUM. Using default value: %s", TestTimeoutMedium) - } - } - if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_LONG"); ok { - if duration, err := time.ParseDuration(value); err == nil { - TestTimeoutLong = duration - } else { - fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_LONG. Using default value: %s", TestTimeoutLong) - } - } - if value, ok := os.LookupEnv("CODEFLARE_TEST_TIMEOUT_GPU_PROVISIONING"); ok { - if duration, err := time.ParseDuration(value); err == nil { - TestTimeoutGpuProvisioning = duration - } else { - fmt.Printf("Error parsing CODEFLARE_TEST_TIMEOUT_GPU_PROVISIONING. Using default value: %s", TestTimeoutGpuProvisioning) - } - } - - // Gomega settings - gomega.SetDefaultEventuallyTimeout(TestTimeoutShort) - gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) - gomega.SetDefaultConsistentlyDuration(30 * time.Second) - gomega.SetDefaultConsistentlyPollingInterval(1 * time.Second) - // Disable object truncation on test results - format.MaxLength = 0 -} diff --git a/test/support/test.go b/test/support/test.go deleted file mode 100644 index 683b0f21..00000000 --- a/test/support/test.go +++ /dev/null @@ -1,139 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "context" - "os" - "path" - "sync" - "testing" - - "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" -) - -type Test interface { - T() *testing.T - Ctx() context.Context - Client() Client - OutputDir() string - - gomega.Gomega - - NewTestNamespace(...Option[*corev1.Namespace]) *corev1.Namespace -} - -type Option[T any] interface { - applyTo(to T) error -} - -type errorOption[T any] func(to T) error - -// nolint: unused -// To be removed when the false-positivity is fixed. -func (o errorOption[T]) applyTo(to T) error { - return o(to) -} - -var _ Option[any] = errorOption[any](nil) - -func With(t *testing.T) Test { - t.Helper() - ctx := context.Background() - if deadline, ok := t.Deadline(); ok { - withDeadline, cancel := context.WithDeadline(ctx, deadline) - t.Cleanup(cancel) - ctx = withDeadline - } - - return &T{ - WithT: gomega.NewWithT(t), - t: t, - ctx: ctx, - } -} - -type T struct { - *gomega.WithT - t *testing.T - // nolint: containedctx - ctx context.Context - client Client - outputDir string - once struct { - client sync.Once - outputDir sync.Once - } -} - -func (t *T) T() *testing.T { - return t.t -} - -func (t *T) Ctx() context.Context { - return t.ctx -} - -func (t *T) Client() Client { - t.T().Helper() - t.once.client.Do(func() { - c, err := newTestClient() - if err != nil { - t.T().Fatalf("Error creating client: %v", err) - } - t.client = c - }) - return t.client -} - -func (t *T) OutputDir() string { - t.T().Helper() - t.once.outputDir.Do(func() { - if parent, ok := os.LookupEnv(CodeFlareTestOutputDir); ok { - if !path.IsAbs(parent) { - if cwd, err := os.Getwd(); err == nil { - // best effort to output the parent absolute path - parent = path.Join(cwd, parent) - } - } - t.T().Logf("Creating output directory in parent directory: %s", parent) - dir, err := os.MkdirTemp(parent, t.T().Name()) - if err != nil { - t.T().Fatalf("Error creating output directory: %v", err) - } - t.outputDir = dir - } else { - t.T().Logf("Creating ephemeral output directory as %s env variable is unset", CodeFlareTestOutputDir) - t.outputDir = t.T().TempDir() - } - t.T().Logf("Output directory has been created at: %s", t.outputDir) - }) - return t.outputDir -} - -func (t *T) NewTestNamespace(options ...Option[*corev1.Namespace]) *corev1.Namespace { - t.T().Helper() - namespace := createTestNamespace(t, options...) - t.T().Cleanup(func() { - storeAllPodLogs(t, namespace) - storeEvents(t, namespace) - deleteTestNamespace(t, namespace) - }) - return namespace -} diff --git a/test/support/utils.go b/test/support/utils.go deleted file mode 100644 index 595ac410..00000000 --- a/test/support/utils.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "io/fs" - "os" - "path" - - "github.com/onsi/gomega" -) - -func Ptr[T any](v T) *T { - return &v -} - -type OutputType string - -const ( - Log OutputType = "log" -) - -func WriteToOutputDir(t Test, fileName string, fileType OutputType, data []byte) { - t.T().Helper() - t.Expect(os.WriteFile(path.Join(t.OutputDir(), fileName+"."+string(fileType)), data, fs.ModePerm)). - To(gomega.Succeed()) -} From 484a3fa58bf6eece2441ae2338228171962818e3 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 13 Oct 2023 15:49:04 +0100 Subject: [PATCH 213/377] updating support package import --- test/e2e/instascale_app_wrapper.go | 3 +-- test/e2e/instascale_machinepool_test.go | 2 +- test/e2e/mnist_pytorch_mcad_job_test.go | 3 +-- test/e2e/mnist_raycluster_sdk_test.go | 3 +-- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 3 +-- test/e2e/support.go | 3 +-- 6 files changed, 6 insertions(+), 11 deletions(-) diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go index 805df1bc..e030817f 100644 --- a/test/e2e/instascale_app_wrapper.go +++ b/test/e2e/instascale_app_wrapper.go @@ -17,14 +17,13 @@ limitations under the License. package e2e import ( + . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/project-codeflare/codeflare-operator/test/support" ) func instaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) *mcadv1beta1.AppWrapper { diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 386b224d..2791cb07 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -20,11 +20,11 @@ import ( "testing" . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - . "github.com/project-codeflare/codeflare-operator/test/support" ) func TestInstascaleMachinePool(t *testing.T) { diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 84722152..7a157c04 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -20,14 +20,13 @@ import ( "testing" . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/project-codeflare/codeflare-operator/test/support" ) // Trains the MNIST dataset as a batch Job managed by MCAD, and asserts successful completion of the training job. diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 29643fe6..b8cc5af7 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -20,6 +20,7 @@ import ( "testing" . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -27,8 +28,6 @@ import ( corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/project-codeflare/codeflare-operator/test/support" ) // Creates a Ray cluster, and trains the MNIST dataset using the CodeFlare SDK. diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index a7956f66..725ced9d 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -21,14 +21,13 @@ import ( "testing" . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/project-codeflare/codeflare-operator/test/support" ) // Trains the MNIST dataset as a RayJob, executed by a Ray cluster managed by MCAD, diff --git a/test/e2e/support.go b/test/e2e/support.go index 14bf19ce..e9165688 100644 --- a/test/e2e/support.go +++ b/test/e2e/support.go @@ -20,8 +20,7 @@ import ( "embed" "github.com/onsi/gomega" - - "github.com/project-codeflare/codeflare-operator/test/support" + "github.com/project-codeflare/codeflare-common/support" ) //go:embed *.py *.txt From f6e75536e1487bc2d757b232340a6ba919561074 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 16 Oct 2023 11:12:25 +0100 Subject: [PATCH 214/377] removing defaults make target --- Makefile | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 2e6ab468..541fba37 100644 --- a/Makefile +++ b/Makefile @@ -125,27 +125,6 @@ help: ## Display this help. ##@ Development -DEFAULTS_TEST_FILE := test/support/defaults.go - -.PHONY: defaults -defaults: - $(info Regenerating $(DEFAULTS_TEST_FILE)) - @echo "package support" > $(DEFAULTS_TEST_FILE) - @echo "" >> $(DEFAULTS_TEST_FILE) - @echo "// ***********************" >> $(DEFAULTS_TEST_FILE) - @echo "// DO NOT EDIT THIS FILE" >> $(DEFAULTS_TEST_FILE) - @echo "// ***********************" >> $(DEFAULTS_TEST_FILE) - @echo "" >> $(DEFAULTS_TEST_FILE) - @echo "const (" >> $(DEFAULTS_TEST_FILE) - @echo " CodeFlareSDKVersion = \"$(CODEFLARE_SDK_VERSION)\"" >> $(DEFAULTS_TEST_FILE) - @echo " RayVersion = \"$(RAY_VERSION)\"" >> $(DEFAULTS_TEST_FILE) - @echo " RayImage = \"$(RAY_IMAGE)\"" >> $(DEFAULTS_TEST_FILE) - @echo "" >> $(DEFAULTS_TEST_FILE) - @echo ")" >> $(DEFAULTS_TEST_FILE) - @echo "" >> $(DEFAULTS_TEST_FILE) - - gofmt -w $(DEFAULTS_TEST_FILE) - # this encounters sed issues on MacOS, quick fix is to use gsed or to escape the parentheses i.e. \( \) .PHONY: manifests manifests: controller-gen kustomize install-yq ## Generate RBAC objects. @@ -173,11 +152,11 @@ modules: ## Update Go dependencies. go mod tidy .PHONY: build -build: modules defaults fmt vet ## Build manager binary. +build: modules fmt vet ## Build manager binary. go build -o bin/manager main.go .PHONY: run -run: modules defaults manifests fmt vet ## Run a controller from your host. +run: modules manifests fmt vet ## Run a controller from your host. go run ./main.go .PHONY: image-build @@ -290,7 +269,7 @@ validate-bundle: install-operator-sdk $(OPERATOR_SDK) bundle validate ./bundle --select-optional suite=operatorframework .PHONY: bundle -bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. +bundle: manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion @@ -369,11 +348,11 @@ catalog-push: ## Push a catalog image. podman push $(CATALOG_IMG) $(CATALOG_PUSH_OPT) .PHONY: test-unit -test-unit: defaults manifests fmt vet envtest ## Run unit tests. +test-unit: manifests fmt vet envtest ## Run unit tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(go list ./... | grep -v /test/) -coverprofile cover.out .PHONY: test-e2e -test-e2e: defaults manifests fmt vet ## Run e2e tests. +test-e2e: manifests fmt vet ## Run e2e tests. go test -timeout 30m -v ./test/e2e .PHONY: kind-e2e From cad078ee681a07d3d0f08a0e97f1b9606ec394d4 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 23 Oct 2023 14:41:42 +0100 Subject: [PATCH 215/377] updating files --- test/e2e/instascale_machinepool_test.go | 1 - test/e2e/instascale_machineset_test.go | 5 +- test/support/client.go | 137 ------------------------ test/support/environment.go | 104 ------------------ test/support/machine.go | 32 ------ test/support/ray.go | 70 ------------ 6 files changed, 2 insertions(+), 347 deletions(-) delete mode 100644 test/support/client.go delete mode 100644 test/support/environment.go delete mode 100644 test/support/machine.go delete mode 100644 test/support/ray.go diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 2791cb07..69fa037d 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -24,7 +24,6 @@ import ( mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ) func TestInstascaleMachinePool(t *testing.T) { diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index fad7717b..827fd404 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -4,11 +4,10 @@ import ( "testing" . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/project-codeflare/codeflare-operator/test/support" ) func TestInstascaleMachineSet(t *testing.T) { @@ -31,7 +30,7 @@ func TestInstascaleMachineSet(t *testing.T) { "mnist.py": ReadFile(test, "mnist.py"), }) - // // Setup batch job and AppWrapper + // Setup batch job and AppWrapper aw := instaScaleJobAppWrapper(test, namespace, cm) // look for machine set with aw name - expect to find it diff --git a/test/support/client.go b/test/support/client.go deleted file mode 100644 index d911c7c2..00000000 --- a/test/support/client.go +++ /dev/null @@ -1,137 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/versioned" - rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" - - "k8s.io/client-go/dynamic" - "k8s.io/client-go/kubernetes" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/client-go/tools/clientcmd" - - imagev1 "github.com/openshift/client-go/image/clientset/versioned" - machinev1 "github.com/openshift/client-go/machine/clientset/versioned" - routev1 "github.com/openshift/client-go/route/clientset/versioned" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. -) - -type Client interface { - Core() kubernetes.Interface - Machine() machinev1.Interface - Route() routev1.Interface - Image() imagev1.Interface - MCAD() mcadclient.Interface - Ray() rayclient.Interface - Dynamic() dynamic.Interface -} - -type testClient struct { - core kubernetes.Interface - machine machinev1.Interface - route routev1.Interface - image imagev1.Interface - mcad mcadclient.Interface - ray rayclient.Interface - dynamic dynamic.Interface -} - -var _ Client = (*testClient)(nil) - -func (t *testClient) Core() kubernetes.Interface { - return t.core -} - -func (t *testClient) Machine() machinev1.Interface { - return t.machine -} - -func (t *testClient) Route() routev1.Interface { - return t.route -} - -func (t *testClient) Image() imagev1.Interface { - return t.image -} -func (t *testClient) MCAD() mcadclient.Interface { - return t.mcad -} - -func (t *testClient) Ray() rayclient.Interface { - return t.ray -} - -func (t *testClient) Dynamic() dynamic.Interface { - return t.dynamic -} - -func newTestClient() (Client, error) { - cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - clientcmd.NewDefaultClientConfigLoadingRules(), - &clientcmd.ConfigOverrides{}, - ).ClientConfig() - if err != nil { - return nil, err - } - - kubeClient, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, err - } - - machineClient, err := machinev1.NewForConfig(cfg) - if err != nil { - return nil, err - } - - routeClient, err := routev1.NewForConfig(cfg) - if err != nil { - return nil, err - } - - imageClient, err := imagev1.NewForConfig(cfg) - if err != nil { - return nil, err - } - - mcadClient, err := mcadclient.NewForConfig(cfg) - if err != nil { - return nil, err - } - - rayClient, err := rayclient.NewForConfig(cfg) - if err != nil { - return nil, err - } - - dynamicClient, err := dynamic.NewForConfig(cfg) - if err != nil { - return nil, err - } - - return &testClient{ - core: kubeClient, - machine: machineClient, - route: routeClient, - image: imageClient, - mcad: mcadClient, - ray: rayClient, - dynamic: dynamicClient, - }, nil -} diff --git a/test/support/environment.go b/test/support/environment.go deleted file mode 100644 index 7d8e6bc8..00000000 --- a/test/support/environment.go +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "os" - "strings" -) - -const ( - // The environment variables hereafter can be used to change the components - // used for testing. - - CodeFlareTestSdkVersion = "CODEFLARE_TEST_SDK_VERSION" - CodeFlareTestRayVersion = "CODEFLARE_TEST_RAY_VERSION" - CodeFlareTestRayImage = "CODEFLARE_TEST_RAY_IMAGE" - CodeFlareTestPyTorchImage = "CODEFLARE_TEST_PYTORCH_IMAGE" - - // The testing output directory, to write output files into. - CodeFlareTestOutputDir = "CODEFLARE_TEST_OUTPUT_DIR" - - // The namespace where a secret containing InstaScale OCM token is stored and the secret name. - InstaScaleOcmSecret = "INSTASCALE_OCM_SECRET" - - // Cluster ID for OSD cluster used in tests, used for testing InstaScale - OsdClusterID = "CLUSTERID" - - // Type of cluster test is run on - ClusterTypeEnvVar = "CLUSTER_TYPE" -) - -type ClusterType string - -const ( - OsdCluster ClusterType = "OSD" - OcpCluster ClusterType = "OCP" - HypershiftCluster ClusterType = "HYPERSHIFT" - UndefinedCluster ClusterType = "UNDEFINED" -) - -func GetCodeFlareSDKVersion() string { - return lookupEnvOrDefault(CodeFlareTestSdkVersion, CodeFlareSDKVersion) -} - -func GetRayVersion() string { - return lookupEnvOrDefault(CodeFlareTestRayVersion, RayVersion) -} - -func GetRayImage() string { - return lookupEnvOrDefault(CodeFlareTestRayImage, RayImage) -} - -func GetPyTorchImage() string { - return lookupEnvOrDefault(CodeFlareTestPyTorchImage, "pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime") -} - -func GetInstascaleOcmSecret() (string, string) { - res := strings.SplitN(lookupEnvOrDefault(InstaScaleOcmSecret, "default/instascale-ocm-secret"), "/", 2) - return res[0], res[1] -} - -func GetOsdClusterId() (string, bool) { - return os.LookupEnv(OsdClusterID) -} - -func GetClusterType(t Test) ClusterType { - clusterType, ok := os.LookupEnv(ClusterTypeEnvVar) - if !ok { - t.T().Logf("Expected environment variable %s not found, cluster type is not defined.", ClusterTypeEnvVar) - return UndefinedCluster - } - switch clusterType { - case "OSD": - return OsdCluster - case "OCP": - return OcpCluster - case "HYPERSHIFT": - return HypershiftCluster - default: - t.T().Logf("Expected environment variable %s contains unexpected value: '%s'", ClusterTypeEnvVar, clusterType) - return UndefinedCluster - } -} - -func lookupEnvOrDefault(key, value string) string { - if v, ok := os.LookupEnv(key); ok { - return v - } - return value -} diff --git a/test/support/machine.go b/test/support/machine.go deleted file mode 100644 index 26acb4bc..00000000 --- a/test/support/machine.go +++ /dev/null @@ -1,32 +0,0 @@ -package support - -import ( - "github.com/onsi/gomega" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - machinev1beta1 "github.com/openshift/api/machine/v1beta1" -) - -func GetMachineSets(t Test) ([]machinev1beta1.MachineSet, error) { - ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms.Items, err -} - -func Machines(t Test, machineSetName string) func(g gomega.Gomega) []machinev1beta1.Machine { - return func(g gomega.Gomega) []machinev1beta1.Machine { - machine, err := t.Client().Machine().MachineV1beta1().Machines("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{LabelSelector: "machine.openshift.io/cluster-api-machineset=" + machineSetName}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return machine.Items - } -} - -func GetMachines(t Test, machineSetName string) []machinev1beta1.Machine { - t.T().Helper() - return Machines(t, machineSetName)(t) -} - -func MachineSetId(machineSet machinev1beta1.MachineSet) string { - return machineSet.Name -} diff --git a/test/support/ray.go b/test/support/ray.go deleted file mode 100644 index f18edaa0..00000000 --- a/test/support/ray.go +++ /dev/null @@ -1,70 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package support - -import ( - "github.com/onsi/gomega" - rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const RayJobDefaultClusterSelectorKey = "ray.io/cluster" - -func RayJob(t Test, namespace, name string) func(g gomega.Gomega) *rayv1.RayJob { - return func(g gomega.Gomega) *rayv1.RayJob { - job, err := t.Client().Ray().RayV1().RayJobs(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return job - } -} - -func GetRayJob(t Test, namespace, name string) *rayv1.RayJob { - t.T().Helper() - return RayJob(t, namespace, name)(t) -} - -func RayJobStatus(job *rayv1.RayJob) rayv1.JobStatus { - return job.Status.JobStatus -} - -func GetRayJobId(t Test, namespace, name string) string { - t.T().Helper() - job := RayJob(t, namespace, name)(t) - return job.Status.JobId -} - -func RayCluster(t Test, namespace, name string) func(g gomega.Gomega) *rayv1.RayCluster { - return func(g gomega.Gomega) *rayv1.RayCluster { - cluster, err := t.Client().Ray().RayV1().RayClusters(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) - g.Expect(err).NotTo(gomega.HaveOccurred()) - return cluster - } -} - -func GetRayCluster(t Test, namespace, name string) *rayv1.RayCluster { - t.T().Helper() - return RayCluster(t, namespace, name)(t) -} - -func RayClusterState(cluster *rayv1.RayCluster) rayv1.ClusterState { - return cluster.Status.State -} - -func WriteRayJobLogs(t Test, rayClient RayClusterClient, namespace, name string) { - WriteRayJobAPILogs(t, rayClient, GetRayJobId(t, namespace, name)) -} From 7c8b0fd13f38506285f9dcc454b37be785606a37 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Tue, 24 Oct 2023 13:16:25 +0100 Subject: [PATCH 216/377] update go.mod and go.sum --- go.mod | 5 +++-- go.sum | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 40254116..912d2a1b 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,8 @@ go 1.19 require ( github.com/onsi/gomega v1.27.10 - github.com/openshift-online/ocm-sdk-go v0.1.368 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 - github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c + github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 github.com/project-codeflare/instascale v0.2.1 github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 @@ -70,6 +69,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect + github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect diff --git a/go.sum b/go.sum index 867260a3..c2efac62 100644 --- a/go.sum +++ b/go.sum @@ -391,6 +391,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 h1:TRMLDP6IYt0CAd3+BkvY/r2lkpjI3sOsxf3tnQojZ9k= +github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= github.com/project-codeflare/instascale v0.2.1 h1:t+Ax/sk4yPQznO6N+U8Cq+bk3afCcZDj9wnHfiGSDBg= github.com/project-codeflare/instascale v0.2.1/go.mod h1:zSzBTP4cFkg+bD4JyYTDmDnGwVKY/+6ACks57NAiscc= github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 h1:hZhGwKTPeHYYhNbvO27NOjozVpy7m3I3apKf81u9U3A= From 1d5c85d366f29a5a898b177765175f3139f8e48e Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Tue, 24 Oct 2023 13:16:59 +0100 Subject: [PATCH 217/377] addressing feedback --- .github/workflows/tag-and-build.yml | 2 -- Makefile | 6 ------ 2 files changed, 8 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 2ecdfb72..54f57a64 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -93,10 +93,8 @@ jobs: - name: Adjust MCAD, SDK and InstaScale dependencies in the code run: | # Remove leading 'v' - CODEFLARE_SDK_VERSION=$(cut -c2- <<< ${{ github.event.inputs.codeflare-sdk-version }}) sed -i -E "s/(.*MCAD_VERSION \?= ).*/\1${{ github.event.inputs.mcad-version }}/" Makefile sed -i -E "s/(.*MCAD_REF \?= ).*/\1release-\${MCAD_VERSION}/" Makefile - sed -i -E "s/(.*CODEFLARE_SDK_VERSION \?= ).*/\1$CODEFLARE_SDK_VERSION/" Makefile sed -i -E "s/(.*INSTASCALE_VERSION \?= ).*/\1${{ github.event.inputs.instascale-version }}/" Makefile - name: Login to Quay.io diff --git a/Makefile b/Makefile index 541fba37..9f8dcbfb 100644 --- a/Makefile +++ b/Makefile @@ -27,9 +27,6 @@ KUBERAY_VERSION ?= v1.0.0-rc.1 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 -# CODEFLARE_SDK_VERSION defines the default version of the CodeFlare SDK -CODEFLARE_SDK_VERSION ?= 0.10.1 - # OPERATORS_REPO_ORG points to GitHub repository organization where bundle PR is opened against # OPERATORS_REPO_FORK_ORG points to GitHub repository fork organization where bundle build is pushed to OPERATORS_REPO_ORG ?= redhat-openshift-ecosystem @@ -64,9 +61,6 @@ IMAGE_ORG_BASE ?= quay.io/project-codeflare # codeflare.dev/codeflare-operator-bundle:$VERSION and codeflare.dev/codeflare-operator-catalog:$VERSION. IMAGE_TAG_BASE ?= $(IMAGE_ORG_BASE)/codeflare-operator -# RAY_IMAGE defines the default container image for Ray (used for testing) -RAY_IMAGE ?= rayproject/ray:$(RAY_VERSION) - # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(VERSION) From ff2a9db5621061d9ef5e259bd7ef4100ecb686a8 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Tue, 24 Oct 2023 11:45:45 -0400 Subject: [PATCH 218/377] Update GRPC version for CVE Signed-off-by: Anish Asthana --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 912d2a1b..51d0a856 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,7 @@ require ( google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/grpc v1.58.2 // indirect + google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index c2efac62..9f8c35a3 100644 --- a/go.sum +++ b/go.sum @@ -865,8 +865,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= -google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From d851f6765a671928ebc80a962aa5220b666ce264 Mon Sep 17 00:00:00 2001 From: Srihari Date: Fri, 20 Oct 2023 17:36:26 +0530 Subject: [PATCH 219/377] Set security context on test pod to avoid pod security warning --- test/e2e/mnist_raycluster_sdk_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index b8cc5af7..aeede52e 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -175,6 +175,16 @@ func TestMNISTRayClusterSDK(t *testing.T) { }, }, WorkingDir: "/workdir", + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: Ptr(false), + SeccompProfile: &corev1.SeccompProfile{ + Type: "RuntimeDefault", + }, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + RunAsNonRoot: Ptr(true), + }, }, }, Volumes: []corev1.Volume{ From 487244c25f3eab06cee534b12cefe11cb93e976e Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Tue, 24 Oct 2023 12:26:23 +0200 Subject: [PATCH 220/377] Use kuberay/ray-operator@vX.Y.Z Go module --- Makefile | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 9f8dcbfb..19776cfb 100644 --- a/Makefile +++ b/Makefile @@ -142,7 +142,7 @@ vet: ## Run go vet against code. modules: ## Update Go dependencies. go get $(MCAD_REPO)@$(MCAD_VERSION) go get $(INSTASCALE_REPO)@$(INSTASCALE_VERSION) - go get github.com/ray-project/kuberay/ray-operator + go get github.com/ray-project/kuberay/ray-operator@$(KUBERAY_VERSION) go mod tidy .PHONY: build diff --git a/go.mod b/go.mod index 51d0a856..124c8211 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 github.com/project-codeflare/instascale v0.2.1 github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 - github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 + github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 k8s.io/apimachinery v0.26.3 diff --git a/go.sum b/go.sum index 9f8c35a3..729858b6 100644 --- a/go.sum +++ b/go.sum @@ -424,8 +424,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15 h1:s+/WzpqoFot/fgo1uq7BLgeiOy3HZuT+UVtCb3h7R+I= -github.com/ray-project/kuberay/ray-operator v0.0.0-20231016183545-097828931d15/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= +github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 h1:aoOajXNG2QnTPmnSzbCXLFBXrVk7bzQGrwwVzTXSOzo= +github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= From 15340def5ea22c3426378a607b836d6e1f32b812 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 31 Oct 2023 14:24:31 +0100 Subject: [PATCH 221/377] Remove automatic triage/needs-triage label creation --- .github/workflows/auto-add-issues.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/auto-add-issues.yml b/.github/workflows/auto-add-issues.yml index a8be31eb..c1e94099 100644 --- a/.github/workflows/auto-add-issues.yml +++ b/.github/workflows/auto-add-issues.yml @@ -4,17 +4,6 @@ on: types: - opened jobs: - add_label: - name: Add needs-triage label to new issues - runs-on: ubuntu-latest - permissions: - issues: write - steps: - - uses: actions/checkout@v3 - - run: gh issue edit ${{ github.event.issue.number }} --add-label "triage/needs-triage" - env: - GH_TOKEN: ${{ github.token }} - add-to-project: name: Add issue to project runs-on: ubuntu-latest From 5c99252a263aa46e07aad4cf1d2c22ae73e07ab6 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 31 Oct 2023 16:25:34 +0000 Subject: [PATCH 222/377] Added context to InstaScale Setup --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 6173b853..5e706576 100644 --- a/main.go +++ b/main.go @@ -154,7 +154,7 @@ func main() { Scheme: mgr.GetScheme(), Config: cfg.InstaScale.InstaScaleConfiguration, } - exitOnError(instaScaleController.SetupWithManager(mgr), "Error setting up InstaScale controller") + exitOnError(instaScaleController.SetupWithManager(context.Background(), mgr), "Error setting up InstaScale controller") } exitOnError(mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping), "unable to set up health check") From b48815e8c66473f834dc121434399455999f7985 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 31 Oct 2023 16:37:41 +0000 Subject: [PATCH 223/377] Update gomod --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 124c8211..932d7614 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 - github.com/project-codeflare/instascale v0.2.1 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 + github.com/project-codeflare/instascale v0.3.0 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 diff --git a/go.sum b/go.sum index 729858b6..c5ad83a7 100644 --- a/go.sum +++ b/go.sum @@ -393,10 +393,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 h1:TRMLDP6IYt0CAd3+BkvY/r2lkpjI3sOsxf3tnQojZ9k= github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= -github.com/project-codeflare/instascale v0.2.1 h1:t+Ax/sk4yPQznO6N+U8Cq+bk3afCcZDj9wnHfiGSDBg= -github.com/project-codeflare/instascale v0.2.1/go.mod h1:zSzBTP4cFkg+bD4JyYTDmDnGwVKY/+6ACks57NAiscc= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1 h1:hZhGwKTPeHYYhNbvO27NOjozVpy7m3I3apKf81u9U3A= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.1/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/instascale v0.3.0 h1:PSlwbqqUsFTkTQ5KUhMFRebfokySnEZwav97xZixLQs= +github.com/project-codeflare/instascale v0.3.0/go.mod h1:IU1Wl+zqTpMpZ49BOcr6U+A6gF3AjcmFdKo9ZwP3TDI= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 h1:dU2Ev0SijdNm30Y9mjdKJL1Fp6l07rnRBKhSbx1kX9g= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From 4930dd50efa66e840f4dce437e4f705c16fa3b69 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 31 Oct 2023 17:37:10 +0000 Subject: [PATCH 224/377] Removed defaults.go reference --- .github/workflows/tag-and-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 54f57a64..a96567c8 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -138,7 +138,7 @@ jobs: uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} - file_pattern: 'README.md */defaults.go *.yaml Makefile go.mod go.sum' + file_pattern: 'README.md *.yaml Makefile go.mod go.sum' create_branch: true branch: ${{ env.PR_BRANCH_NAME }} From 1c7f2290cb6226aa8eda6d8f33ae748d3227a700 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 31 Oct 2023 18:32:35 +0000 Subject: [PATCH 225/377] Update dependency versions for release v1.0.0 --- Makefile | 4 ++-- README.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 19776cfb..f62aa93a 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.2.1 +INSTASCALE_VERSION ?= v0.3.0 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.37.1 +MCAD_VERSION ?= v1.38.0 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} diff --git a/README.md b/README.md index a21c1647..5cac0031 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.0.0-rc.4](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0-rc.4) | -| Multi-Cluster App Dispatcher | [v1.37.1](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.37.1) | -| CodeFlare-SDK | [v0.10.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.10.1) | -| InstaScale | [v0.2.1](https://github.com/project-codeflare/instascale/releases/tag/v0.2.1) | +| CodeFlare Operator | [v1.0.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0) | +| Multi-Cluster App Dispatcher | [v1.38.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.38.0) | +| CodeFlare-SDK | [v0.11.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.11.0) | +| InstaScale | [v0.3.0](https://github.com/project-codeflare/instascale/releases/tag/v0.3.0) | | KubeRay | [v0.5.0](https://github.com/ray-project/kuberay/releases/tag/v0.5.0) | From fd87dcd8969ffc1936b1af8dcd6e69a368648815 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 1 Nov 2023 10:33:01 +0000 Subject: [PATCH 226/377] Updated KubeRay version in Readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5cac0031..d8cdd53d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ CodeFlare Stack Compatibility Matrix | Multi-Cluster App Dispatcher | [v1.38.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.38.0) | | CodeFlare-SDK | [v0.11.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.11.0) | | InstaScale | [v0.3.0](https://github.com/project-codeflare/instascale/releases/tag/v0.3.0) | -| KubeRay | [v0.5.0](https://github.com/ray-project/kuberay/releases/tag/v0.5.0) | +| KubeRay | [v1.0.0-rc.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0-rc.0) | ## Development From 2e3d952a2802d6940394818db7299c3d0519ba13 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 1 Nov 2023 16:35:59 +0100 Subject: [PATCH 227/377] Fix broken catalog-build-from-index command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f62aa93a..dff50cba 100644 --- a/Makefile +++ b/Makefile @@ -331,7 +331,7 @@ catalog-build-from-index: opm ## Build a catalog image. mkdir catalog $(OPM) render $(CATALOG_BASE_IMG) -o yaml > catalog/bundles.yaml $(OPM) render $(BUNDLE_IMG) $(OPM_BUNDLE_OPT) > catalog/codeflare-operator-bundle.yaml - $(SED) -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml + $(SED) -i -E "s/(.*)(- name: codeflare-operator.v0.2.0)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml $(OPM) validate catalog $(OPM) generate dockerfile catalog podman build . -f catalog.Dockerfile -t $(CATALOG_IMG) From 719b6bdb4c868f4ec512373738665a0d187e3b06 Mon Sep 17 00:00:00 2001 From: Kevin Date: Mon, 23 Oct 2023 10:07:03 -0400 Subject: [PATCH 228/377] remove scorecard gh action Signed-off-by: Kevin --- .github/workflows/olm_scorecard.yml | 28 ---------------------------- Makefile | 4 ---- 2 files changed, 32 deletions(-) delete mode 100644 .github/workflows/olm_scorecard.yml diff --git a/.github/workflows/olm_scorecard.yml b/.github/workflows/olm_scorecard.yml deleted file mode 100644 index ba83b801..00000000 --- a/.github/workflows/olm_scorecard.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Operator Scorecard Test - -on: [push, pull_request] - -jobs: - scorecard-check: - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v2 - - - name: Setup and start KinD cluster - uses: ./.github/actions/kind - - - name: Build bundle - run: make bundle-build - - - name: Install yq - run: | - sudo wget -O /usr/bin/yq https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64 - sudo chmod +x /usr/bin/yq - - - name: Modify scorecard config - run: | - yq e 'del(.stages[].tests[] | select(.labels.test == "olm-crds-have-resources-test"))' -i bundle/tests/scorecard/config.yaml - - - name: Run Operator SDK Scorecard - run: make scorecard-bundle diff --git a/Makefile b/Makefile index dff50cba..a52a4e79 100644 --- a/Makefile +++ b/Makefile @@ -364,7 +364,3 @@ imports: openshift-goimports ## Organize imports in go files using openshift-goi .PHONY: verify-imports verify-imports: openshift-goimports ## Run import verifications. ./hack/verify-imports.sh $(OPENSHIFT-GOIMPORTS) - -.PHONY: scorecard-bundle -scorecard-bundle: install-operator-sdk ## Run scorecard tests on bundle image. - $(OPERATOR_SDK) scorecard bundle From 544d1e7bafbf709b0cc0bc41e1d60ffce9a0d831 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 1 Nov 2023 13:12:26 +0100 Subject: [PATCH 229/377] Regenerate CFO CRD manifests for release --- .github/workflows/tag-and-build.yml | 5 ++++- config/crd/mcad/kustomization.yaml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index a96567c8..f9812930 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -92,7 +92,6 @@ jobs: - name: Adjust MCAD, SDK and InstaScale dependencies in the code run: | - # Remove leading 'v' sed -i -E "s/(.*MCAD_VERSION \?= ).*/\1${{ github.event.inputs.mcad-version }}/" Makefile sed -i -E "s/(.*MCAD_REF \?= ).*/\1release-\${MCAD_VERSION}/" Makefile sed -i -E "s/(.*INSTASCALE_VERSION \?= ).*/\1${{ github.event.inputs.instascale-version }}/" Makefile @@ -134,6 +133,10 @@ jobs: run: | go mod tidy + - name: Regenerate CFO CRD manifests + run: | + make manifests + - name: Commit changes in the code back to repository uses: stefanzweifel/git-auto-commit-action@v4 with: diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index e53d2dd9..23063b67 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.37.1 +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.38.0 From bfadb47830846a307be2a3ce7b30eb0726178bee Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 2 Nov 2023 16:23:52 -0400 Subject: [PATCH 230/377] Use CGO_ENABLED=1 for FIPS compliance reasons Signed-off-by: Anish Asthana --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ab721099..93f3f332 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ COPY pkg/ pkg/ # Build USER root -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go +RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o manager main.go FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8 WORKDIR / From 57568b53384fc386d9e92119e8f2223425b5f383 Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Mon, 6 Nov 2023 16:53:38 -0500 Subject: [PATCH 231/377] Add stricttag to pass fips check warnings Signed-off-by: Anish Asthana --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 93f3f332..ea02d1b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ COPY pkg/ pkg/ # Build USER root -RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o manager main.go +RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -tags strictfipsruntime -a -o manager main.go FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8 WORKDIR / From 3885aef496d62a4d220ef2105c3a01a044b6042e Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 24 Oct 2023 17:24:27 +0100 Subject: [PATCH 232/377] Added launch.json for local debugging with VSCode --- .gitignore | 3 +++ .vscode/launch.json | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.gitignore b/.gitignore index 898d0779..3126962c 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ bundle.Dockerfile *~ .DS_STORE + +#VSCode +.vscode diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..7357bfc7 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug CodeFlare Operator", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/main.go", + "env": { + "KUBECONFIG": "", + "NAMESPACE": "", + }, + "showLog": true + }, + ] + } From 25b40b24aa0f465081e22b823180cb0992c20beb Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 24 Oct 2023 17:33:46 +0100 Subject: [PATCH 233/377] Fixed Json --- .vscode/launch.json | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 7357bfc7..731487da 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,17 +1,17 @@ { - "version": "0.2.0", - "configurations": [ - { - "name": "Debug CodeFlare Operator", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/main.go", - "env": { - "KUBECONFIG": "", - "NAMESPACE": "", - }, - "showLog": true + "version": "0.2.0", + "configurations": [ + { + "name": "Debug CodeFlare Operator", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/main.go", + "env": { + "KUBECONFIG": "", + "NAMESPACE": "" }, - ] - } + "showLog": true + } + ] +} From cea4ceb4fd3542fc1513f35632f653122ca78674 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 25 Oct 2023 09:30:01 +0100 Subject: [PATCH 234/377] Added instructions to CONTRIBUTING.md --- CONTRIBUTING.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 83edd2f9..df63e179 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,3 +56,9 @@ To write and inspect unit tests: - MCAD and InstaScale unit tests under `mcad_controller_test.go` and `instascale_controller_test.go` in the `controllers` dir - Unit test functions are defined in `suite_test.go` (with utils in `util/util.go`) in the `controllers dir` - Test cases defined under `controllers/testdata` + + ## Local debugging with VSCode + Steps outlining how to run the operator locally. + - Populate the `.vscode/launch.json` file with the location of your Kubernetes config file and desired namespace. + - In VSCode on the activity bar click `Run and Debug` or `CTRL + SHIFT + D` to start a local debugging session of the CodeFlare Operator. + The operator should be running as intended. From b0f15b871219902fb80c4575b0a04707a372208c Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 1 Nov 2023 10:56:50 +0000 Subject: [PATCH 235/377] Review changes links, additional info --- .gitignore | 2 +- CONTRIBUTING.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 3126962c..9be9b01b 100644 --- a/.gitignore +++ b/.gitignore @@ -30,5 +30,5 @@ bundle.Dockerfile .DS_STORE -#VSCode +# VSCode .vscode diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df63e179..9fd4a3f8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,6 +59,7 @@ To write and inspect unit tests: ## Local debugging with VSCode Steps outlining how to run the operator locally. - - Populate the `.vscode/launch.json` file with the location of your Kubernetes config file and desired namespace. + - Ensure you are authenticated to your Kubernetes/OpenShift Cluster. + - Populate the [.vscode/launch.json](https://github.com/project-codeflare/codeflare-operator/tree/main/.vscode/launch.json) file with the location of your Kubernetes config file and desired namespace. - In VSCode on the activity bar click `Run and Debug` or `CTRL + SHIFT + D` to start a local debugging session of the CodeFlare Operator. The operator should be running as intended. From 5f826e0003633a8d88558ada2390bc221f8f55f0 Mon Sep 17 00:00:00 2001 From: Shilpa Chugh Date: Tue, 7 Nov 2023 13:43:59 +0530 Subject: [PATCH 236/377] Update workflow to add kuberay in compatibility matrix --- .github/workflows/tag-and-build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index f9812930..ed7b1769 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -24,6 +24,10 @@ on: description: 'Published version of InstaScale' required: true default: 'v0.0.0-dev' + kuberay-version: + description: 'Published version of KubeRay' + required: true + default: 'v0.0.0-dev' is-stable: description: 'Select if the built image should be tagged as stable' required: true @@ -89,6 +93,7 @@ jobs: sed -i -E "s|(.*Multi-Cluster App Dispatcher.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.mcad-version }}\2${{ github.event.inputs.mcad-version }}\3|" README.md sed -i -E "s|(.*CodeFlare-SDK.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3|" README.md sed -i -E "s|(.*InstaScale.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.instascale-version }}\2${{ github.event.inputs.instascale-version }}\3|" README.md + sed -i -E "s|(.*KubeRay.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kuberay-version }}\2${{ github.event.inputs.kuberay-version }}\3|" README.md - name: Adjust MCAD, SDK and InstaScale dependencies in the code run: | From e95759a6fea725760a6a8a9dbfd7331fd0b4bf08 Mon Sep 17 00:00:00 2001 From: Shilpa Chugh Date: Tue, 7 Nov 2023 15:56:58 +0530 Subject: [PATCH 237/377] change description --- .github/workflows/tag-and-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index ed7b1769..827a33c6 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -25,7 +25,7 @@ on: required: true default: 'v0.0.0-dev' kuberay-version: - description: 'Published version of KubeRay' + description: 'Tested version of KubeRay' required: true default: 'v0.0.0-dev' is-stable: From 72cace9ed1adbd91a22675516e237dd94f2d4d79 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 8 Nov 2023 13:47:46 +0100 Subject: [PATCH 238/377] Enable TestMNISTRayClusterSDK test --- .github/actions/kind/action.yml | 16 ++ go.mod | 2 +- go.sum | 4 +- test/e2e/mnist_raycluster_sdk.py | 24 ++- test/e2e/mnist_raycluster_sdk_test.go | 151 ++++++++---------- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 4 +- 6 files changed, 107 insertions(+), 94 deletions(-) diff --git a/.github/actions/kind/action.yml b/.github/actions/kind/action.yml index 59dcafef..f76e60af 100644 --- a/.github/actions/kind/action.yml +++ b/.github/actions/kind/action.yml @@ -1,6 +1,12 @@ name: "Set up KinD" description: "Step to start and configure KinD cluster" +inputs: + kind-node-hostname: + description: "Hostname of the main kind node" + required: false + default: kind + runs: using: "composite" steps: @@ -56,3 +62,13 @@ runs: curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/"${VERSION}"/deploy/static/provider/kind/deploy.yaml | sed "s/--publish-status-address=localhost/--report-node-internal-ip-address\\n - --status-update-interval=10/g" | kubectl apply -f - kubectl annotate ingressclass nginx "ingressclass.kubernetes.io/is-default-class=true" kubectl -n ingress-nginx wait --timeout=300s --for=condition=Available deployments --all + + - name: Add ${{ inputs.kind-node-hostname }} host to machine hosts + shell: bash + run: echo "127.0.0.1 ${{ inputs.kind-node-hostname }}" | sudo tee -a /etc/hosts + + - name: Set env variables for tests to properly leverage KinD cluster + shell: bash + run: | + echo "CLUSTER_TYPE=KIND" >> $GITHUB_ENV + echo "CLUSTER_HOSTNAME=${{ inputs.kind-node-hostname }}" >> $GITHUB_ENV diff --git a/go.mod b/go.mod index 932d7614..98011af7 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 - github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 + github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb github.com/project-codeflare/instascale v0.3.0 github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 diff --git a/go.sum b/go.sum index c5ad83a7..42334303 100644 --- a/go.sum +++ b/go.sum @@ -391,8 +391,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16 h1:TRMLDP6IYt0CAd3+BkvY/r2lkpjI3sOsxf3tnQojZ9k= -github.com/project-codeflare/codeflare-common v0.0.0-20231023092720-93d03492db16/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= +github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb h1:L2Gdr2SlvshDKZY2KK6507AwzQ1NSfRbMQuz5dOsYNM= +github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= github.com/project-codeflare/instascale v0.3.0 h1:PSlwbqqUsFTkTQ5KUhMFRebfokySnEZwav97xZixLQs= github.com/project-codeflare/instascale v0.3.0/go.mod h1:IU1Wl+zqTpMpZ49BOcr6U+A6gF3AjcmFdKo9ZwP3TDI= github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 h1:dU2Ev0SijdNm30Y9mjdKJL1Fp6l07rnRBKhSbx1kX9g= diff --git a/test/e2e/mnist_raycluster_sdk.py b/test/e2e/mnist_raycluster_sdk.py index b830a004..cb3c0af5 100644 --- a/test/e2e/mnist_raycluster_sdk.py +++ b/test/e2e/mnist_raycluster_sdk.py @@ -1,4 +1,5 @@ import sys +import os from time import sleep @@ -8,17 +9,38 @@ from codeflare_sdk.job.jobs import DDPJobDefinition namespace = sys.argv[1] +ray_image = os.getenv('RAY_IMAGE') +host = os.getenv('CLUSTER_HOSTNAME') + +ingress_options = {} +if host is not None: + ingress_options = { + "ingresses": [ + { + "ingressName": "ray-dashboard", + "port": 8265, + "pathType": "Prefix", + "path": "/", + "host": host, + }, + ] + } + cluster = Cluster(ClusterConfiguration( name='mnist', namespace=namespace, num_workers=1, + head_cpus='500m', + head_memory=2, min_cpus='500m', max_cpus=1, min_memory=0.5, - max_memory=1, + max_memory=2, num_gpus=0, instascale=False, + image=ray_image, + ingress_options=ingress_options, )) cluster.up() diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index aeede52e..015cbfc5 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -40,104 +40,59 @@ func TestMNISTRayClusterSDK(t *testing.T) { test := With(t) test.T().Parallel() - // Currently blocked by https://github.com/project-codeflare/codeflare-sdk/pull/251 , remove the skip once SDK with the PR is released - test.T().Skip("Requires https://github.com/project-codeflare/codeflare-sdk/pull/251") - // Create a namespace namespace := test.NewTestNamespace() // Test configuration - config := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist-raycluster-sdk", - Namespace: namespace.Name, + config := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // SDK script + "mnist_raycluster_sdk.py": ReadFile(test, "mnist_raycluster_sdk.py"), + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }) + + // Create RBAC, retrieve token for user with limited rights + policyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, }, - BinaryData: map[string][]byte{ - // SDK script - "mnist_raycluster_sdk.py": ReadFile(test, "mnist_raycluster_sdk.py"), - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, }, - Immutable: Ptr(true), - } - config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - - // SDK client RBAC - serviceAccount := &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ServiceAccount", + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk-user", - Namespace: namespace.Name, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"ingresses"}, }, } - serviceAccount, err = test.Client().Core().CoreV1().ServiceAccounts(namespace.Name).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - role := &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{ - APIVersion: rbacv1.SchemeGroupVersion.String(), - Kind: "Role", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", - Namespace: namespace.Name, - }, - Rules: []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1.GroupVersion.Group}, - Resources: []string{"rayclusters", "rayclusters/status"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"route.openshift.io"}, - Resources: []string{"routes"}, - }, + // Create cluster wide RBAC, required for SDK OpenShift check + // TODO reevaluate once SDK change OpenShift detection logic + clusterPolicyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"config.openshift.io"}, + Resources: []string{"ingresses"}, + ResourceNames: []string{"cluster"}, }, } - role, err = test.Client().Core().RbacV1().Roles(namespace.Name).Create(test.Ctx(), role, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - roleBinding := &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: rbacv1.SchemeGroupVersion.String(), - Kind: "RoleBinding", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: rbacv1.SchemeGroupVersion.Group, - Kind: "Role", - Name: role.Name, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - APIGroup: corev1.SchemeGroupVersion.Group, - Name: serviceAccount.Name, - Namespace: serviceAccount.Namespace, - }, - }, - } - _, err = test.Client().Core().RbacV1().RoleBindings(namespace.Name).Create(test.Ctx(), roleBinding, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) + sa := CreateServiceAccount(test, namespace.Name) + role := CreateRole(test, namespace.Name, policyRules) + CreateRoleBinding(test, namespace.Name, sa, role) + clusterRole := CreateClusterRole(test, clusterPolicyRules) + CreateClusterRoleBinding(test, sa, clusterRole) job := &batchv1.Job{ TypeMeta: metav1.TypeMeta{ @@ -161,7 +116,8 @@ func TestMNISTRayClusterSDK(t *testing.T) { // See https://github.com/project-codeflare/codeflare-sdk/pull/146 Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", Env: []corev1.EnvVar{ - corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "RAY_IMAGE", Value: GetRayImage()}, }, Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, VolumeMounts: []corev1.VolumeMount{ @@ -206,12 +162,31 @@ func TestMNISTRayClusterSDK(t *testing.T) { }, }, RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: serviceAccount.Name, + ServiceAccountName: sa.Name, }, }, }, } - job, err = test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) + if GetClusterType(test) == KindCluster { + // Take first KinD node and redirect pod hostname requests there + node := GetNodes(test)[0] + hostname := GetClusterHostname(test) + IP := GetNodeInternalIP(test, node) + + test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP) + job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{ + { + IP: IP, + Hostnames: []string{hostname}, + }, + } + + // Propagate hostname into Python code as env variable + hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname} + job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar) + } + + job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 725ced9d..b8d3f4d0 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -108,7 +108,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("1G"), + corev1.ResourceMemory: resource.MustParse("2G"), }, }, VolumeMounts: []corev1.VolumeMount{ @@ -168,7 +168,7 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("1G"), + corev1.ResourceMemory: resource.MustParse("2G"), }, }, }, From 4366322e7e89422ca32af488c35cb372c2f93aff Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 30 Oct 2023 15:53:44 +0100 Subject: [PATCH 239/377] Setup existing DW tests to run in CFO --- test/odh/environment.go | 49 +++++++ test/odh/mcad_ray_test.go | 98 ++++++++++++++ test/odh/notebook.go | 99 ++++++++++++++ test/odh/pytorch_mcad_test.go | 77 +++++++++++ test/odh/resources/custom-nb-small.yaml | 165 +++++++++++++++++++++++ test/odh/resources/mnist.py | 160 ++++++++++++++++++++++ test/odh/resources/mnist_mcad_mini.ipynb | 93 +++++++++++++ test/odh/resources/mnist_ray_mini.ipynb | 145 ++++++++++++++++++++ test/odh/resources/requirements.txt | 4 + test/odh/support.go | 34 +++++ 10 files changed, 924 insertions(+) create mode 100644 test/odh/environment.go create mode 100644 test/odh/mcad_ray_test.go create mode 100644 test/odh/notebook.go create mode 100644 test/odh/pytorch_mcad_test.go create mode 100644 test/odh/resources/custom-nb-small.yaml create mode 100644 test/odh/resources/mnist.py create mode 100644 test/odh/resources/mnist_mcad_mini.ipynb create mode 100644 test/odh/resources/mnist_ray_mini.ipynb create mode 100644 test/odh/resources/requirements.txt create mode 100644 test/odh/support.go diff --git a/test/odh/environment.go b/test/odh/environment.go new file mode 100644 index 00000000..0a087c9d --- /dev/null +++ b/test/odh/environment.go @@ -0,0 +1,49 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "os" + + . "github.com/project-codeflare/codeflare-common/support" +) + +const ( + // The environment variable for namespace where ODH is installed to. + odhNamespaceEnvVar = "ODH_NAMESPACE" + // The environment variable for ODH Notebook ImageStream name + notebookImageStreamName = "NOTEBOOK_IMAGE_STREAM_NAME" +) + +func GetOpenDataHubNamespace() string { + return lookupEnvOrDefault(odhNamespaceEnvVar, "opendatahub") +} + +func GetNotebookImageStreamName(t Test) string { + isName, ok := os.LookupEnv(notebookImageStreamName) + if !ok { + t.T().Fatalf("Expected environment variable %s not found, please use this environment variable to specify what ImageStream to use for Notebook.", notebookImageStreamName) + } + return isName +} + +func lookupEnvOrDefault(key, value string) string { + if v, ok := os.LookupEnv(key); ok { + return v + } + return value +} diff --git a/test/odh/mcad_ray_test.go b/test/odh/mcad_ray_test.go new file mode 100644 index 00000000..770b64d9 --- /dev/null +++ b/test/odh/mcad_ray_test.go @@ -0,0 +1,98 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "testing" + + . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + + rbacv1 "k8s.io/api/rbac/v1" +) + +func TestMCADRay(t *testing.T) { + test := With(t) + + // Create a namespace + namespace := test.NewTestNamespace() + + // Test configuration + jupyterNotebookConfigMapFileName := "mnist_ray_mini.ipynb" + config := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // MNIST Ray Notebook + jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_ray_mini.ipynb"), + "mnist.py": ReadFile(test, "resources/mnist.py"), + "requirements.txt": ReadFile(test, "resources/requirements.txt"), + }) + + // Create RBAC, retrieve token for user with limited rights + policyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1alpha1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, + }, + } + + // Create cluster wide RBAC, required for SDK OpenShift check + // TODO reevaluate once SDK change OpenShift detection logic + clusterPolicyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"config.openshift.io"}, + Resources: []string{"ingresses"}, + ResourceNames: []string{"cluster"}, + }, + } + + sa := CreateServiceAccount(test, namespace.Name) + role := CreateRole(test, namespace.Name, policyRules) + CreateRoleBinding(test, namespace.Name, sa, role) + clusterRole := CreateClusterRole(test, clusterPolicyRules) + CreateClusterRoleBinding(test, sa, clusterRole) + token := CreateToken(test, namespace.Name, sa) + + // Create Notebook CR + createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) + + // Make sure the AppWrapper is created and running + test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). + Should( + And( + HaveLen(1), + ContainElement(WithTransform(AppWrapperName, HavePrefix("mnisttest"))), + ContainElement(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))), + ), + ) + + // Make sure the AppWrapper finishes and is deleted + test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). + Should(HaveLen(0)) +} diff --git a/test/odh/notebook.go b/test/odh/notebook.go new file mode 100644 index 00000000..8c7b2827 --- /dev/null +++ b/test/odh/notebook.go @@ -0,0 +1,99 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "bytes" + "html/template" + + gomega "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/yaml" + + imagev1 "github.com/openshift/api/image/v1" +) + +const recommendedTagAnnotation = "opendatahub.io/workbench-image-recommended" + +var notebookResource = schema.GroupVersionResource{Group: "kubeflow.org", Version: "v1", Resource: "notebooks"} + +type NotebookProps struct { + IngressDomain string + OpenShiftApiUrl string + KubernetesBearerToken string + Namespace string + OpenDataHubNamespace string + ImageStreamName string + ImageStreamTag string + NotebookConfigMapName string + NotebookConfigMapFileName string + NotebookPVC string +} + +func createNotebook(test Test, namespace *corev1.Namespace, notebookToken, jupyterNotebookConfigMapName, jupyterNotebookConfigMapFileName string) { + // Create PVC for Notebook + notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "10Gi", corev1.ReadWriteOnce) + + // Retrieve ImageStream tag for + is := GetImageStream(test, GetOpenDataHubNamespace(), GetNotebookImageStreamName(test)) + recommendedTagName := getRecommendedImageStreamTag(test, is) + + // Read the Notebook CR from resources and perform replacements for custom values using go template + notebookProps := NotebookProps{ + IngressDomain: GetOpenShiftIngressDomain(test), + OpenShiftApiUrl: GetOpenShiftApiUrl(test), + KubernetesBearerToken: notebookToken, + Namespace: namespace.Name, + OpenDataHubNamespace: GetOpenDataHubNamespace(), + ImageStreamName: GetNotebookImageStreamName(test), + ImageStreamTag: recommendedTagName, + NotebookConfigMapName: jupyterNotebookConfigMapName, + NotebookConfigMapFileName: jupyterNotebookConfigMapFileName, + NotebookPVC: notebookPVC.Name, + } + notebookTemplate, err := files.ReadFile("resources/custom-nb-small.yaml") + test.Expect(err).NotTo(gomega.HaveOccurred()) + parsedNotebookTemplate, err := template.New("notebook").Parse(string(notebookTemplate)) + test.Expect(err).NotTo(gomega.HaveOccurred()) + + // Filter template and store results to the buffer + notebookBuffer := new(bytes.Buffer) + err = parsedNotebookTemplate.Execute(notebookBuffer, notebookProps) + test.Expect(err).NotTo(gomega.HaveOccurred()) + + // Create Notebook CR + notebookCR := &unstructured.Unstructured{} + err = yaml.NewYAMLOrJSONDecoder(notebookBuffer, 8192).Decode(notebookCR) + test.Expect(err).NotTo(gomega.HaveOccurred()) + _, err = test.Client().Dynamic().Resource(notebookResource).Namespace(namespace.Name).Create(test.Ctx(), notebookCR, metav1.CreateOptions{}) + test.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func getRecommendedImageStreamTag(test Test, is *imagev1.ImageStream) (tagName string) { + for _, tag := range is.Spec.Tags { + if tag.Annotations[recommendedTagAnnotation] == "true" { + return tag.Name + } + } + test.T().Fatalf("tag with annotation '%s' not found in ImageStream %s", recommendedTagAnnotation, is.Name) + return +} diff --git a/test/odh/pytorch_mcad_test.go b/test/odh/pytorch_mcad_test.go new file mode 100644 index 00000000..0dd33a36 --- /dev/null +++ b/test/odh/pytorch_mcad_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "testing" + + . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + + rbacv1 "k8s.io/api/rbac/v1" +) + +func TestMnistPyTorchMCAD(t *testing.T) { + test := With(t) + + // Create a namespace + namespace := test.NewTestNamespace() + + // Test configuration + jupyterNotebookConfigMapFileName := "mnist_mcad_mini.ipynb" + config := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // MNIST MCAD Notebook + jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_mcad_mini.ipynb"), + }) + + // Create RBAC, retrieve token for user with limited rights + policyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + // Needed for job.logs() + { + Verbs: []string{"get"}, + APIGroups: []string{""}, + Resources: []string{"pods/log"}, + }, + } + sa := CreateServiceAccount(test, namespace.Name) + role := CreateRole(test, namespace.Name, policyRules) + CreateRoleBinding(test, namespace.Name, sa, role) + token := CreateToken(test, namespace.Name, sa) + + // Create Notebook CR + createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) + + // Make sure the AppWrapper is created and running + test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). + Should( + And( + HaveLen(1), + ContainElement(WithTransform(AppWrapperName, HavePrefix("mnistjob"))), + ContainElement(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))), + ), + ) + + // Make sure the AppWrapper finishes and is deleted + test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). + Should(HaveLen(0)) +} diff --git a/test/odh/resources/custom-nb-small.yaml b/test/odh/resources/custom-nb-small.yaml new file mode 100644 index 00000000..95aaaf10 --- /dev/null +++ b/test/odh/resources/custom-nb-small.yaml @@ -0,0 +1,165 @@ +# This template maybe used to spin up a custom notebook image +# i.e.: sed s/{{.IngressDomain}}/$(oc get ingresses.config/cluster -o jsonpath={.spec.domain})/g tests/resources/custom-nb.template | oc apply -f - +# resources generated: +# pod/jupyter-nb-kube-3aadmin-0 +# service/jupyter-nb-kube-3aadmin +# route.route.openshift.io/jupyter-nb-kube-3aadmin (jupyter-nb-kube-3aadmin-opendatahub.apps.tedbig412.cp.fyre.ibm.com) +# service/jupyter-nb-kube-3aadmin-tls +apiVersion: kubeflow.org/v1 +kind: Notebook +metadata: + annotations: + notebooks.opendatahub.io/inject-oauth: "true" + notebooks.opendatahub.io/last-image-selection: codeflare-notebook:latest + notebooks.opendatahub.io/last-size-selection: Small + notebooks.opendatahub.io/oauth-logout-url: https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}/notebookController/kube-3aadmin/home + opendatahub.io/link: https://jupyter-nb-kube-3aadmin-{{.Namespace}}.{{.IngressDomain}}/notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin + opendatahub.io/username: kube:admin + generation: 1 + labels: + app: jupyter-nb-kube-3aadmin + opendatahub.io/dashboard: "true" + opendatahub.io/odh-managed: "true" + opendatahub.io/user: kube-3aadmin + name: jupyter-nb-kube-3aadmin + namespace: {{.Namespace}} +spec: + template: + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: nvidia.com/gpu.present + operator: NotIn + values: + - "true" + weight: 1 + containers: + - env: + - name: NOTEBOOK_ARGS + value: |- + --ServerApp.port=8888 + --ServerApp.token='' + --ServerApp.password='' + --ServerApp.base_url=/notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin + --ServerApp.quit_button=False + --ServerApp.tornado_settings={"user":"kube-3aadmin","hub_host":"https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}","hub_prefix":"/notebookController/kube-3aadmin"} + - name: JUPYTER_IMAGE + value: image-registry.openshift-image-registry.svc:5000/{{.OpenDataHubNamespace}}/{{.ImageStreamName}}:{{.ImageStreamTag}} + - name: JUPYTER_NOTEBOOK_PORT + value: "8888" + - name: OCP_SERVER + value: {{.OpenShiftApiUrl}} + - name: OCP_TOKEN + value: {{.KubernetesBearerToken}} + image: image-registry.openshift-image-registry.svc:5000/{{.OpenDataHubNamespace}}/{{.ImageStreamName}}:{{.ImageStreamTag}} + command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{.NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{.Namespace}} && sleep infinity"] + # args: ["pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/mcad.ipynb /opt/app-root/src/mcad-out.ipynb" ] + imagePullPolicy: Always + # livenessProbe: + # failureThreshold: 3 + # httpGet: + # path: /notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin/api + # port: notebook-port + # scheme: HTTP + # initialDelaySeconds: 10 + # periodSeconds: 5 + # successThreshold: 1 + # timeoutSeconds: 1 + name: jupyter-nb-kube-3aadmin + ports: + - containerPort: 8888 + name: notebook-port + protocol: TCP + resources: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 3Gi + volumeMounts: + - mountPath: /opt/app-root/src + name: jupyterhub-nb-kube-3aadmin-pvc + - mountPath: /opt/app-root/notebooks + name: {{.NotebookConfigMapName}} + workingDir: /opt/app-root/src + - args: + - --provider=openshift + - --https-address=:8443 + - --http-address= + - --openshift-service-account=jupyter-nb-kube-3aadmin + - --cookie-secret-file=/etc/oauth/config/cookie_secret + - --cookie-expire=24h0m0s + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --upstream=http://localhost:8888 + - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - --skip-auth-regex=^(?:/notebook/$(NAMESPACE)/jupyter-nb-kube-3aadmin)?/api$ + - --email-domain=* + - --skip-provider-button + - --openshift-sar={"verb":"get","resource":"notebooks","resourceAPIGroup":"kubeflow.org","resourceName":"jupyter-nb-kube-3aadmin","namespace":"$(NAMESPACE)"} + - --logout-url=https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}/notebookController/kube-3aadmin/home + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.redhat.io/openshift4/ose-oauth-proxy:v4.10 + imagePullPolicy: Always + livenessProbe: + failureThreshold: 3 + httpGet: + path: /oauth/healthz + port: oauth-proxy + scheme: HTTPS + initialDelaySeconds: 30 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + name: oauth-proxy + ports: + - containerPort: 8443 + name: oauth-proxy + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /oauth/healthz + port: oauth-proxy + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 100m + memory: 64Mi + volumeMounts: + - mountPath: /etc/oauth/config + name: oauth-config + - mountPath: /etc/tls/private + name: tls-certificates + enableServiceLinks: false + serviceAccountName: jupyter-nb-kube-3aadmin + volumes: + - name: jupyterhub-nb-kube-3aadmin-pvc + persistentVolumeClaim: + claimName: {{.NotebookPVC}} + - name: oauth-config + secret: + defaultMode: 420 + secretName: jupyter-nb-kube-3aadmin-oauth-config + - name: tls-certificates + secret: + defaultMode: 420 + secretName: jupyter-nb-kube-3aadmin-tls + - name: {{.NotebookConfigMapName}} + configMap: + name: {{.NotebookConfigMapName}} diff --git a/test/odh/resources/mnist.py b/test/odh/resources/mnist.py new file mode 100644 index 00000000..d6a21194 --- /dev/null +++ b/test/odh/resources/mnist.py @@ -0,0 +1,160 @@ +# Copyright 2022 IBM, Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch +from pytorch_lightning import LightningModule, Trainer +from pytorch_lightning.callbacks.progress import TQDMProgressBar +from torch import nn +from torch.nn import functional as F +from torch.utils.data import DataLoader, random_split +from torchmetrics import Accuracy +from torchvision import transforms +from torchvision.datasets import MNIST + +PATH_DATASETS = os.environ.get("PATH_DATASETS", ".") +BATCH_SIZE = 256 if torch.cuda.is_available() else 64 +# %% + +print("prior to running the trainer") +print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) +print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) + + +class LitMNIST(LightningModule): + def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4): + + super().__init__() + + # Set our init args as class attributes + self.data_dir = data_dir + self.hidden_size = hidden_size + self.learning_rate = learning_rate + + # Hardcode some dataset specific attributes + self.num_classes = 10 + self.dims = (1, 28, 28) + channels, width, height = self.dims + self.transform = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)), + ] + ) + + # Define PyTorch model + self.model = nn.Sequential( + nn.Flatten(), + nn.Linear(channels * width * height, hidden_size), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden_size, hidden_size), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden_size, self.num_classes), + ) + + self.val_accuracy = Accuracy() + self.test_accuracy = Accuracy() + + def forward(self, x): + x = self.model(x) + return F.log_softmax(x, dim=1) + + def training_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + return loss + + def validation_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + preds = torch.argmax(logits, dim=1) + self.val_accuracy.update(preds, y) + + # Calling self.log will surface up scalars for you in TensorBoard + self.log("val_loss", loss, prog_bar=True) + self.log("val_acc", self.val_accuracy, prog_bar=True) + + def test_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + preds = torch.argmax(logits, dim=1) + self.test_accuracy.update(preds, y) + + # Calling self.log will surface up scalars for you in TensorBoard + self.log("test_loss", loss, prog_bar=True) + self.log("test_acc", self.test_accuracy, prog_bar=True) + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) + return optimizer + + #################### + # DATA RELATED HOOKS + #################### + + def prepare_data(self): + # download + print("Downloading MNIST dataset...") + MNIST(self.data_dir, train=True, download=True) + MNIST(self.data_dir, train=False, download=True) + + def setup(self, stage=None): + + # Assign train/val datasets for use in dataloaders + if stage == "fit" or stage is None: + mnist_full = MNIST(self.data_dir, train=True, transform=self.transform) + self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000]) + + # Assign test dataset for use in dataloader(s) + if stage == "test" or stage is None: + self.mnist_test = MNIST( + self.data_dir, train=False, transform=self.transform + ) + + def train_dataloader(self): + return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) + + def val_dataloader(self): + return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) + + def test_dataloader(self): + return DataLoader(self.mnist_test, batch_size=BATCH_SIZE) + + +# Init DataLoader from MNIST Dataset + +model = LitMNIST() + +print("GROUP: ", int(os.environ.get("GROUP_WORLD_SIZE", 1))) +print("LOCAL: ", int(os.environ.get("LOCAL_WORLD_SIZE", 1))) + +# Initialize a trainer +trainer = Trainer( + accelerator="auto", + # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs + max_epochs=2, + callbacks=[TQDMProgressBar(refresh_rate=20)], + num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), + devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), + strategy="ddp", +) + +# Train the model ⚡ +trainer.fit(model) diff --git a/test/odh/resources/mnist_mcad_mini.ipynb b/test/odh/resources/mnist_mcad_mini.ipynb new file mode 100644 index 00000000..0b53324a --- /dev/null +++ b/test/odh/resources/mnist_mcad_mini.ipynb @@ -0,0 +1,93 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Import pieces from codeflare-sdk\n", + "from codeflare_sdk.job.jobs import DDPJobDefinition\n", + "from time import sleep" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47ca5c15", + "metadata": { + "tags": ["parameters"] + }, + "outputs": [], + "source": [ + "#parameters\n", + "namespace = \"default\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26b21373", + "metadata": {}, + "outputs": [], + "source": [ + "job = DDPJobDefinition(name=\"mnistjob\", script=\"mnist.py\", scheduler_args={\"namespace\": namespace}, j=\"1x1\", gpu=0, cpu=1, memMB=2000, image=\"quay.io/project-codeflare/mnist-job-test:v0.0.1\").submit()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d24e9f95", + "metadata": {}, + "outputs": [], + "source": [ + "finished = False\n", + "while not finished:\n", + " sleep(1)\n", + " try:\n", + " finished = (\"Epoch 4: 100%\" in job.logs())\n", + " except:\n", + " finished = False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f078b7cd", + "metadata": {}, + "outputs": [], + "source": [ + "job.cancel()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "vscode": { + "interpreter": { + "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/odh/resources/mnist_ray_mini.ipynb b/test/odh/resources/mnist_ray_mini.ipynb new file mode 100644 index 00000000..38992cc7 --- /dev/null +++ b/test/odh/resources/mnist_ray_mini.ipynb @@ -0,0 +1,145 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Import pieces from codeflare-sdk\n", + "from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration\n", + "from codeflare_sdk.job.jobs import DDPJobDefinition\n", + "from time import sleep" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30888aed", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#parameters\n", + "namespace = \"default\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f4bc870-091f-4e11-9642-cba145710159", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Create our cluster and submit appwrapper\n", + "cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', head_cpus=1, head_memory=2, num_workers=1, min_cpus=1, max_cpus=1, min_memory=1, max_memory=2, num_gpus=0, instascale=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0884bbc-c224-4ca0-98a0-02dfa09c2200", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Bring up the cluster\n", + "cluster.up()\n", + "cluster.wait_ready()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df71c1ed", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "cluster.status()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7fd45bc5-03c0-4ae5-9ec5-dd1c30f1a084", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "cluster.details()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47ca5c15", + "metadata": {}, + "outputs": [], + "source": [ + "job = DDPJobDefinition(name=\"mnisttest\", script=\"mnist.py\", workspace=\"file:///opt/app-root/notebooks/..data\", scheduler_args={\"requirements\": \"/opt/app-root/notebooks/requirements.txt\"}).submit(cluster)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f63a178a", + "metadata": {}, + "outputs": [], + "source": [ + "finished = False\n", + "while not finished:\n", + " sleep(1)\n", + " status = job.status()\n", + " finished = (str(status.state) == \"SUCCEEDED\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b099777", + "metadata": {}, + "outputs": [], + "source": [ + "cluster.down()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "vscode": { + "interpreter": { + "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/test/odh/resources/requirements.txt b/test/odh/resources/requirements.txt new file mode 100644 index 00000000..7266b064 --- /dev/null +++ b/test/odh/resources/requirements.txt @@ -0,0 +1,4 @@ +pytorch_lightning==1.5.10 +ray_lightning +torchmetrics==0.9.1 +torchvision==0.12.0 diff --git a/test/odh/support.go b/test/odh/support.go new file mode 100644 index 00000000..d828ed95 --- /dev/null +++ b/test/odh/support.go @@ -0,0 +1,34 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "embed" + + "github.com/onsi/gomega" + "github.com/project-codeflare/codeflare-common/support" +) + +//go:embed resources/* +var files embed.FS + +func ReadFile(t support.Test, fileName string) []byte { + t.T().Helper() + file, err := files.ReadFile(fileName) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return file +} From 6d4a5cd4088cb1204ed54f619d978bf497947a0e Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Mon, 13 Nov 2023 16:43:57 +0200 Subject: [PATCH 240/377] Add build info to logs --- Makefile | 14 +++++++++++++- main.go | 7 +++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index a52a4e79..564fbb07 100644 --- a/Makefile +++ b/Makefile @@ -97,6 +97,18 @@ endif SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec +BUILD_DATE := $(shell date +%Y-%m-%d\ %H:%M) +BUILD_TAG_SHA := $(shell git rev-list --abbrev-commit --tags --max-count=1) +BUILD_TAG_NAME := $(shell git describe --abbrev=0 --tags ${BUILD_TAG_SHA} 2>/dev/null || true) +BUILD_SHA := $(shell git rev-parse --short HEAD) +BUILD_VERSION := $(BUILD_TAG_NAME:v%=%) +ifneq ($(BUILD_SHA), $(BUILD_TAG_SHA)) + BUILD_VERSION := $(BUILD_VERSION)-$(BUILD_SHA) +endif +ifneq ($(shell git status --porcelain),) + BUILD_VERSION := $(BUILD_VERSION)-dirty +endif + .PHONY: all all: build @@ -147,7 +159,7 @@ modules: ## Update Go dependencies. .PHONY: build build: modules fmt vet ## Build manager binary. - go build -o bin/manager main.go + go build -ldflags "-X 'main.BuildVersion=$(BUILD_VERSION)' -X 'main.BuildDate=$(BUILD_DATE)'" -o bin/manager main.go .PHONY: run run: modules manifests fmt vet ## Run a controller from your host. diff --git a/main.go b/main.go index 5e706576..6387f18f 100644 --- a/main.go +++ b/main.go @@ -58,8 +58,10 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + BuildVersion = "UNKNOWN" + BuildDate = "UNKNOWN" ) func init() { @@ -85,6 +87,7 @@ func main() { zapOptions.BindFlags(flag.CommandLine) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) + setupLog.Info("Build info", "version", BuildVersion, "date", BuildDate) ctx := ctrl.SetupSignalHandler() From 32c66716ec87bcfcdf0ddf904febd198253c5499 Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Tue, 14 Nov 2023 11:53:54 +0200 Subject: [PATCH 241/377] Log MCAD and InstaScale versions --- Makefile | 9 ++++++++- main.go | 17 ++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 564fbb07..7d647462 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,14 @@ modules: ## Update Go dependencies. .PHONY: build build: modules fmt vet ## Build manager binary. - go build -ldflags "-X 'main.BuildVersion=$(BUILD_VERSION)' -X 'main.BuildDate=$(BUILD_DATE)'" -o bin/manager main.go + go build \ + -ldflags " \ + -X 'main.OperatorVersion=$(BUILD_VERSION)' \ + -X 'main.McadVersion=$(MCAD_VERSION)' \ + -X 'main.InstaScaleVersion=$(INSTASCALE_VERSION)' \ + -X 'main.BuildDate=$(BUILD_DATE)' \ + " \ + -o bin/manager main.go .PHONY: run run: modules manifests fmt vet ## Run a controller from your host. diff --git a/main.go b/main.go index 6387f18f..84827f43 100644 --- a/main.go +++ b/main.go @@ -58,10 +58,12 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") - BuildVersion = "UNKNOWN" - BuildDate = "UNKNOWN" + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + OperatorVersion = "UNKNOWN" + McadVersion = "UNKNOWN" + InstaScaleVersion = "UNKNOWN" + BuildDate = "UNKNOWN" ) func init() { @@ -87,7 +89,12 @@ func main() { zapOptions.BindFlags(flag.CommandLine) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) - setupLog.Info("Build info", "version", BuildVersion, "date", BuildDate) + setupLog.Info("Build info", + "operatorVersion", OperatorVersion, + "mcadVersion", McadVersion, + "instaScaleVersion", InstaScaleVersion, + "date", BuildDate, + ) ctx := ctrl.SetupSignalHandler() From 881946ee9a701fa58b29c94ccdcb7681ab6c1767 Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Tue, 14 Nov 2023 14:47:05 +0200 Subject: [PATCH 242/377] Parse command line flags --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 84827f43..4b2ce428 100644 --- a/main.go +++ b/main.go @@ -81,6 +81,7 @@ func main() { flag.StringVar(&configMapName, "config", "codeflare-operator-config", "The name of the ConfigMap to load the operator configuration from. "+ "If it does not exist, the operator will create and initialise it.") + flag.Parse() zapOptions := zap.Options{ Development: true, From 6b06d33aa5b29d824769cf76a3495241a6734e08 Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Tue, 14 Nov 2023 14:48:19 +0200 Subject: [PATCH 243/377] Update github.com/golang/glog v1.1.0 -> v1.1.2 This fixes vmodule panic ref: https://groups.google.com/g/golang-nuts/c/Atlr8uAjn6U/m/iId17Td5BQAJ --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 98011af7..e60aaafe 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect - github.com/golang/glog v1.1.0 // indirect + github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/cel-go v0.12.6 // indirect diff --git a/go.sum b/go.sum index 42334303..54fcdff5 100644 --- a/go.sum +++ b/go.sum @@ -159,8 +159,8 @@ github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5 github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= From f818a6b32cfd4536a95ed809809aeb27571fd7d9 Mon Sep 17 00:00:00 2001 From: Ronen Schaffer Date: Wed, 15 Nov 2023 09:33:45 +0200 Subject: [PATCH 244/377] Parse Zap flags --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 4b2ce428..bf7f7454 100644 --- a/main.go +++ b/main.go @@ -81,13 +81,13 @@ func main() { flag.StringVar(&configMapName, "config", "codeflare-operator-config", "The name of the ConfigMap to load the operator configuration from. "+ "If it does not exist, the operator will create and initialise it.") - flag.Parse() zapOptions := zap.Options{ Development: true, TimeEncoder: zapcore.TimeEncoderOfLayout(time.RFC3339), } zapOptions.BindFlags(flag.CommandLine) + flag.Parse() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) setupLog.Info("Build info", From 7776a0311a4338d81ba97862eda733b3004f575c Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 8 Nov 2023 15:18:15 +0000 Subject: [PATCH 245/377] Changed namespace in MCAD Ray CRB --- config/rbac/mcad-controller-ray-clusterrolebinding.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml index a3931da0..9ab3599e 100644 --- a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml +++ b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml @@ -5,7 +5,7 @@ metadata: subjects: - kind: ServiceAccount name: codeflare-operator-controller-manager - namespace: openshift-operators + namespace: system roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole From 5789a4d52b44956ab9c543e6a9b9f329d70a004d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 16 Nov 2023 14:50:40 +0100 Subject: [PATCH 246/377] e2e: Fix deployment namespace --- config/e2e/kustomization.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml index faf613e7..e06123b2 100644 --- a/config/e2e/kustomization.yaml +++ b/config/e2e/kustomization.yaml @@ -1,3 +1,5 @@ +namespace: openshift-operators + bases: - ../default From 9c0ceccfc960c0acf5595100959b7f209e4b27e3 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Thu, 16 Nov 2023 12:46:18 +0000 Subject: [PATCH 247/377] Update dependency versions for release v1.0.1 --- Makefile | 4 ++-- README.md | 10 +++++----- config/crd/mcad/kustomization.yaml | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 7d647462..d32fdc01 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.3.0 +INSTASCALE_VERSION ?= v0.3.1 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.38.0 +MCAD_VERSION ?= v1.38.1 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} diff --git a/README.md b/README.md index d8cdd53d..7a0280fa 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.0.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.0) | -| Multi-Cluster App Dispatcher | [v1.38.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.38.0) | -| CodeFlare-SDK | [v0.11.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.11.0) | -| InstaScale | [v0.3.0](https://github.com/project-codeflare/instascale/releases/tag/v0.3.0) | -| KubeRay | [v1.0.0-rc.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0-rc.0) | +| CodeFlare Operator | [v1.0.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.1) | +| Multi-Cluster App Dispatcher | [v1.38.1](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.38.1) | +| CodeFlare-SDK | [v0.12.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.12.1) | +| InstaScale | [v0.3.1](https://github.com/project-codeflare/instascale/releases/tag/v0.3.1) | +| KubeRay | [v0.0.0-dev](https://github.com/opendatahub-io/kuberay/releases/tag/v0.0.0-dev) | ## Development diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index 23063b67..3cd50c48 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.38.0 +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.38.1 diff --git a/go.mod b/go.mod index e60aaafe..fd23755d 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb - github.com/project-codeflare/instascale v0.3.0 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 + github.com/project-codeflare/instascale v0.3.1 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 k8s.io/api v0.26.3 diff --git a/go.sum b/go.sum index 54fcdff5..6c33548b 100644 --- a/go.sum +++ b/go.sum @@ -393,10 +393,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb h1:L2Gdr2SlvshDKZY2KK6507AwzQ1NSfRbMQuz5dOsYNM= github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= -github.com/project-codeflare/instascale v0.3.0 h1:PSlwbqqUsFTkTQ5KUhMFRebfokySnEZwav97xZixLQs= -github.com/project-codeflare/instascale v0.3.0/go.mod h1:IU1Wl+zqTpMpZ49BOcr6U+A6gF3AjcmFdKo9ZwP3TDI= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0 h1:dU2Ev0SijdNm30Y9mjdKJL1Fp6l07rnRBKhSbx1kX9g= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= +github.com/project-codeflare/instascale v0.3.1 h1:LIKo5NaX7kDPmAYy1aAkF0yykql3ZdCiiLsBOC4HRcM= +github.com/project-codeflare/instascale v0.3.1/go.mod h1:Fdy3daVhz3BHFvPyr9zfH6uFF2yww73U41Wq2S2zDI8= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 h1:6ILHYAFxDkAnQu3CJebGQPQGcmcy7/E/AhRiea6yOTc= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From 42cce2d0713e4a22a00d145fbac14f67ab3f3016 Mon Sep 17 00:00:00 2001 From: Srihari Date: Tue, 3 Oct 2023 18:28:35 +0530 Subject: [PATCH 248/377] Update SDK e2e test to reflect poetry changes Add Ingress domain for sdk e2e test Revert "Add Ingress domain for sdk e2e test" This reverts commit ffc200474502e2aeebdce1654c8c1a2bce086c25. --- test/e2e/install-codeflare-sdk.sh | 22 ++++++++++++++++++++++ test/e2e/mnist_raycluster_sdk_test.go | 14 +++++++++++++- test/e2e/support.go | 2 +- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100755 test/e2e/install-codeflare-sdk.sh diff --git a/test/e2e/install-codeflare-sdk.sh b/test/e2e/install-codeflare-sdk.sh new file mode 100755 index 00000000..e90f4071 --- /dev/null +++ b/test/e2e/install-codeflare-sdk.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# go to codeflare-sdk folder and install codeflare-sdk +cd .. + +# Install Poetry and configure virtualenvs +pip install poetry +poetry config virtualenvs.create false + +cd codeflare-sdk +# Clone the CodeFlare SDK repository +git clone --branch main https://github.com/project-codeflare/codeflare-sdk.git + +cd codeflare-sdk + +# Lock dependencies and install them +poetry lock --no-update +poetry install --with test,docs + +# Return to the workdir +cd ../.. +cd workdir diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go index 015cbfc5..be6933f6 100644 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ b/test/e2e/mnist_raycluster_sdk_test.go @@ -51,6 +51,8 @@ func TestMNISTRayClusterSDK(t *testing.T) { "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), // MNIST training script "mnist.py": ReadFile(test, "mnist.py"), + // codeflare-sdk installation script + "install-codeflare-sdk.sh": ReadFile(test, "install-codeflare-sdk.sh"), }) // Create RBAC, retrieve token for user with limited rights @@ -119,12 +121,16 @@ func TestMNISTRayClusterSDK(t *testing.T) { {Name: "PYTHONUSERBASE", Value: "/workdir"}, {Name: "RAY_IMAGE", Value: GetRayImage()}, }, - Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_raycluster_sdk.py" + " " + namespace.Name}, + Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python mnist_raycluster_sdk.py" + " " + namespace.Name}, VolumeMounts: []corev1.VolumeMount{ { Name: "test", MountPath: "/test", }, + { + Name: "codeflare-sdk", + MountPath: "/codeflare-sdk", + }, { Name: "workdir", MountPath: "/workdir", @@ -154,6 +160,12 @@ func TestMNISTRayClusterSDK(t *testing.T) { }, }, }, + { + Name: "codeflare-sdk", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, { Name: "workdir", VolumeSource: corev1.VolumeSource{ diff --git a/test/e2e/support.go b/test/e2e/support.go index e9165688..8fc86295 100644 --- a/test/e2e/support.go +++ b/test/e2e/support.go @@ -23,7 +23,7 @@ import ( "github.com/project-codeflare/codeflare-common/support" ) -//go:embed *.py *.txt +//go:embed *.py *.txt *.sh var files embed.FS func ReadFile(t support.Test, fileName string) []byte { From ee0e762cc9f938b4931fc046b8c82e8774e77a1b Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 21 Nov 2023 16:08:58 +0100 Subject: [PATCH 249/377] Add missing KubeRay version parameter for release workflow --- .github/workflows/project-codeflare-release.yml | 5 ++++- .github/workflows/tag-and-build.yml | 1 + README.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 1f8bb0c7..77d0a0a1 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -19,6 +19,9 @@ on: instascale-version: description: 'Version of InstaScale to be released (for example: v0.0.0)' required: true + kuberay-version: + description: 'Tested version of KubeRay (for example: v0.0.0)' + required: true is-stable: description: 'Select if the built images should be tagged as stable' required: true @@ -156,7 +159,7 @@ jobs: steps: - name: Release CodeFlare operator run: | - gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field mcad-version=${{ github.event.inputs.mcad-version }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field instascale-version=${{ github.event.inputs.instascale-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} + gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field mcad-version=${{ github.event.inputs.mcad-version }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field instascale-version=${{ github.event.inputs.instascale-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 827a33c6..b97deb33 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -100,6 +100,7 @@ jobs: sed -i -E "s/(.*MCAD_VERSION \?= ).*/\1${{ github.event.inputs.mcad-version }}/" Makefile sed -i -E "s/(.*MCAD_REF \?= ).*/\1release-\${MCAD_VERSION}/" Makefile sed -i -E "s/(.*INSTASCALE_VERSION \?= ).*/\1${{ github.event.inputs.instascale-version }}/" Makefile + sed -i -E "s/(.*KUBERAY_VERSION \?= ).*/\1${{ github.event.inputs.kuberay-version }}/" Makefile - name: Login to Quay.io uses: redhat-actions/podman-login@v1 diff --git a/README.md b/README.md index 7a0280fa..c3048b29 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ CodeFlare Stack Compatibility Matrix | Multi-Cluster App Dispatcher | [v1.38.1](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.38.1) | | CodeFlare-SDK | [v0.12.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.12.1) | | InstaScale | [v0.3.1](https://github.com/project-codeflare/instascale/releases/tag/v0.3.1) | -| KubeRay | [v0.0.0-dev](https://github.com/opendatahub-io/kuberay/releases/tag/v0.0.0-dev) | +| KubeRay | [v1.0.0-rc.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0-rc.0) | ## Development From 4f04b9f18f260fc6d59506bcf5c2a244a4b7c029 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 21 Nov 2023 14:17:00 -0500 Subject: [PATCH 250/377] set klog to the controller runtime logger Signed-off-by: Kevin --- go.mod | 2 +- main.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fd23755d..455963b0 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( k8s.io/apimachinery v0.26.3 k8s.io/client-go v0.26.3 k8s.io/component-base v0.26.2 + k8s.io/klog/v2 v2.90.1 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 sigs.k8s.io/controller-runtime v0.14.6 sigs.k8s.io/yaml v1.3.0 @@ -114,7 +115,6 @@ require ( k8s.io/apiextensions-apiserver v0.26.1 // indirect k8s.io/apiserver v0.26.2 // indirect k8s.io/klog v1.0.0 // indirect - k8s.io/klog/v2 v2.90.1 // indirect k8s.io/kms v0.26.2 // indirect k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d // indirect k8s.io/metrics v0.26.2 // indirect diff --git a/main.go b/main.go index bf7f7454..5e2bb6e3 100644 --- a/main.go +++ b/main.go @@ -42,6 +42,7 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/client-go/rest" configv1alpha1 "k8s.io/component-base/config/v1alpha1" + "k8s.io/klog/v2" "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -90,6 +91,8 @@ func main() { flag.Parse() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions))) + klog.SetLogger(ctrl.Log) + setupLog.Info("Build info", "operatorVersion", OperatorVersion, "mcadVersion", McadVersion, From 426a67fa701508c11aabb21dbd41eb3aa0383d75 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 23 Nov 2023 08:31:33 +0100 Subject: [PATCH 251/377] Raise SDK e2e test timeout in Python script --- test/e2e/mnist_raycluster_sdk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/mnist_raycluster_sdk.py b/test/e2e/mnist_raycluster_sdk.py index cb3c0af5..d171c9b0 100644 --- a/test/e2e/mnist_raycluster_sdk.py +++ b/test/e2e/mnist_raycluster_sdk.py @@ -62,7 +62,7 @@ done = False time = 0 -timeout = 300 +timeout = 900 while not done: status = job.status() if is_terminal(status.state): From 4de7141a011a8069e22d083402c2a67d5a49825a Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 24 Nov 2023 10:49:05 +0100 Subject: [PATCH 252/377] Adjust release GH workflow to handle error cases --- .github/workflows/tag-and-build.yml | 38 ++++++++++++++++++----------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index b97deb33..28d27c99 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -121,20 +121,6 @@ jobs: podman tag quay.io/${{ github.event.inputs.quay-organization }}/codeflare-operator:${{ github.event.inputs.version }} quay.io/${{ github.event.inputs.quay-organization }}/codeflare-operator:stable make image-push -e IMG=quay.io/${{ github.event.inputs.quay-organization }}/codeflare-operator:stable - - name: Build bundle and create PR in OpenShift community operators repository - run: | - git config --global user.email "138894154+codeflare-machine-account@users.noreply.github.com" - git config --global user.name "codeflare-machine-account" - make openshift-community-operator-release - env: - VERSION: ${{ github.event.inputs.version }} - PREVIOUS_VERSION: ${{ github.event.inputs.replaces }} - INSTASCALE_VERSION: ${{ github.event.inputs.instascale-version }} - MCAD_VERSION: ${{ github.event.inputs.mcad-version }} - GH_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} - OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} - - name: Cleanup the go.mod and go.sum run: | go mod tidy @@ -144,6 +130,7 @@ jobs: make manifests - name: Commit changes in the code back to repository + id: create-pr-branch uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} @@ -159,12 +146,21 @@ jobs: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - name: Wait until PR with code changes is merged + id: wait-pr-merge run: | timeout 3600 bash -c 'until [[ $(gh pr view '${{ env.PR_BRANCH_NAME }}' --json state --jq .state) == "MERGED" ]]; do sleep 5 && echo "$(gh pr view '${{ env.PR_BRANCH_NAME }}' --json state --jq .state)"; done' env: GITHUB_TOKEN: ${{ github.TOKEN }} + - name: Close PR if PR merge failed + if: always() && (steps.wait-pr-merge.outcome == 'failure' || steps.wait-pr-merge.outcome == 'cancelled') + run: | + gh pr close ${{ env.PR_BRANCH_NAME }} + env: + GITHUB_TOKEN: ${{ github.TOKEN }} + - name: Delete remote branch + if: always() && steps.create-pr-branch.outcome == 'success' run: | git push origin --delete ${{ env.PR_BRANCH_NAME }} @@ -180,3 +176,17 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash + + - name: Build bundle and create PR in OpenShift community operators repository + run: | + git config --global user.email "138894154+codeflare-machine-account@users.noreply.github.com" + git config --global user.name "codeflare-machine-account" + make openshift-community-operator-release + env: + VERSION: ${{ github.event.inputs.version }} + PREVIOUS_VERSION: ${{ github.event.inputs.replaces }} + INSTASCALE_VERSION: ${{ github.event.inputs.instascale-version }} + MCAD_VERSION: ${{ github.event.inputs.mcad-version }} + GH_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} + OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} From 0acdfde08f9d8d7dc407ac5f329c0a3fe181d1a1 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 24 Nov 2023 17:01:15 +0100 Subject: [PATCH 253/377] Remove accidental prefix for service account for mcad-controller-ray-clusterrolebinding --- config/rbac/mcad-controller-ray-clusterrolebinding.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml index 9ab3599e..da9e8c02 100644 --- a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml +++ b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml @@ -4,7 +4,7 @@ metadata: name: mcad-controller-ray-clusterrolebinding subjects: - kind: ServiceAccount - name: codeflare-operator-controller-manager + name: controller-manager namespace: system roleRef: apiGroup: rbac.authorization.k8s.io From 39e942246be34ac421cf0e3e9f4e186595216c90 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 24 Nov 2023 15:22:01 +0100 Subject: [PATCH 254/377] Send notification on Slack if e2e fails on push event --- .github/workflows/e2e_tests.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 64d4567e..b20ff968 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -93,3 +93,12 @@ jobs: retention-days: 10 path: | ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log + + - name: Post notification about failure to a Slack channel in case of push event + if: failure() && github.event_name == 'push' + uses: slackapi/slack-github-action@v1.24.0 + with: + channel-id: "codeflare-nightlies" + slack-message: "e2e test on push failed, " + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} From 23bffcc83574cd09fcd5e70c0a917de8ab191e3c Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 29 Nov 2023 12:48:00 +0100 Subject: [PATCH 255/377] Add completion status for TestMNISTPyTorchMCAD --- go.mod | 2 +- go.sum | 4 ++-- test/e2e/mnist_pytorch_mcad_job_test.go | 13 +++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 455963b0..e9f358d4 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 - github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb + github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 github.com/project-codeflare/instascale v0.3.1 github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 diff --git a/go.sum b/go.sum index 6c33548b..9af7ce38 100644 --- a/go.sum +++ b/go.sum @@ -391,8 +391,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb h1:L2Gdr2SlvshDKZY2KK6507AwzQ1NSfRbMQuz5dOsYNM= -github.com/project-codeflare/codeflare-common v0.0.0-20231110155354-042fb171fcdb/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= +github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 h1:81+ma1mchF/LtAGsf+poAt50kJ/fLYjoTAcZOxci1Yc= +github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= github.com/project-codeflare/instascale v0.3.1 h1:LIKo5NaX7kDPmAYy1aAkF0yykql3ZdCiiLsBOC4HRcM= github.com/project-codeflare/instascale v0.3.1/go.mod h1:Fdy3daVhz3BHFvPyr9zfH6uFF2yww73U41Wq2S2zDI8= github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 h1:6ILHYAFxDkAnQu3CJebGQPQGcmcy7/E/AhRiea6yOTc= diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 7a157c04..cbe55fed 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -143,7 +143,8 @@ func TestMNISTPyTorchMCAD(t *testing.T) { }, }, }, - GenericTemplate: Raw(test, job), + GenericTemplate: Raw(test, job), + CompletionStatus: "Complete", }, }, }, @@ -159,13 +160,13 @@ func TestMNISTPyTorchMCAD(t *testing.T) { Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong).Should( Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)), + WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateFailed)), )) // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + test.Expect(GetAppWrapper(test, namespace, aw.Name)). + To(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) } From 148d43ff98ea9195739f7c5e87e601bca9d28cc3 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 29 Nov 2023 16:04:53 +0100 Subject: [PATCH 256/377] Use dedicated KinD composite action from CodeFlare common --- .github/actions/kind/action.yml | 74 -------------------------------- .github/resources-kind/kind.yaml | 31 ------------- .github/workflows/e2e_tests.yaml | 9 +++- .github/workflows/olm_tests.yaml | 9 +++- 4 files changed, 16 insertions(+), 107 deletions(-) delete mode 100644 .github/actions/kind/action.yml delete mode 100644 .github/resources-kind/kind.yaml diff --git a/.github/actions/kind/action.yml b/.github/actions/kind/action.yml deleted file mode 100644 index f76e60af..00000000 --- a/.github/actions/kind/action.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: "Set up KinD" -description: "Step to start and configure KinD cluster" - -inputs: - kind-node-hostname: - description: "Hostname of the main kind node" - required: false - default: kind - -runs: - using: "composite" - steps: - - name: Init directories - shell: bash - run: | - TEMP_DIR="$(pwd)/tmp" - mkdir -p "${TEMP_DIR}" - echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV - - mkdir -p "$(pwd)/bin" - echo "$(pwd)/bin" >> $GITHUB_PATH - - - name: Container image registry - shell: bash - run: | - podman run -d -p 5000:5000 --name registry registry:2.8.1 - - export REGISTRY_ADDRESS=$(hostname -i):5000 - echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV - echo "Container image registry started at ${REGISTRY_ADDRESS}" - - KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml - echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV - envsubst < .github/resources-kind/kind.yaml > ${KIND_CONFIG_FILE} - - sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <> $GITHUB_ENV - echo "CLUSTER_HOSTNAME=${{ inputs.kind-node-hostname }}" >> $GITHUB_ENV diff --git a/.github/resources-kind/kind.yaml b/.github/resources-kind/kind.yaml deleted file mode 100644 index 4546589b..00000000 --- a/.github/resources-kind/kind.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# --------------------------------------------------------------------------- -# Copyright 2023. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# --------------------------------------------------------------------------- - -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: - - role: control-plane - image: kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1 - kubeadmConfigPatches: - - | - kind: InitConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: "ingress-ready=true" -containerdConfigPatches: - - |- - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${REGISTRY_ADDRESS}"] - endpoint = ["http://${REGISTRY_ADDRESS}"] diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index b20ff968..1962a8b8 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -35,6 +35,13 @@ jobs: with: submodules: recursive + - name: Checkout common repo code + uses: actions/checkout@v3 + with: + repository: 'project-codeflare/codeflare-common' + ref: 'main' + path: 'common' + - name: Set Go uses: actions/setup-go@v3 with: @@ -46,7 +53,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Setup and start KinD cluster - uses: ./.github/actions/kind + uses: ./common/github-actions/kind - name: Deploy CodeFlare stack id: deploy diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 382c95f3..83175503 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -34,6 +34,13 @@ jobs: with: fetch-depth: 0 # fetching also previous commits to get tags + - name: Checkout common repo code + uses: actions/checkout@v3 + with: + repository: 'project-codeflare/codeflare-common' + ref: 'main' + path: 'common' + - name: Set Go uses: actions/setup-go@v3 with: @@ -45,7 +52,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Setup and start KinD cluster - uses: ./.github/actions/kind + uses: ./common/github-actions/kind - name: Install OLM run: | From 68c5f67817694252fc76ff68a117b8a01a3f7c3a Mon Sep 17 00:00:00 2001 From: Srihari Date: Thu, 14 Sep 2023 13:22:33 +0530 Subject: [PATCH 257/377] Create OLM upgrade e2e scenario using codeflare SDK --- .github/workflows/olm_tests.yaml | 17 ++ test/e2e/mnist_rayjob.py | 44 +++++ test/e2e/olm_upgrade_test.go | 322 +++++++++++++++++++++++++++++++ test/e2e/raycluster_sdk.py | 29 +++ 4 files changed, 412 insertions(+) create mode 100644 test/e2e/mnist_rayjob.py create mode 100644 test/e2e/olm_upgrade_test.go create mode 100644 test/e2e/raycluster_sdk.py diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 83175503..de1de85b 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -121,6 +121,14 @@ jobs: OPM_BUNDLE_OPT: "--use-http" BUNDLE_PUSH_OPT: "--tls-verify=false" CATALOG_PUSH_OPT: "--tls-verify=false" + - name: Run OLM Upgrade e2e Ray cluster Up test + run: | + export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} + echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV + set -euo pipefail + go test -timeout 30m -v ./test/e2e -run TestMNISTRayClusterUp -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + env: + RUN_OLM_TESTS: true - name: Update Operator to the built version run: | @@ -151,6 +159,15 @@ jobs: SUBSCRIPTION_NAME: "codeflare-operator" SUBSCRIPTION_NAMESPACE: "openshift-operators" + - name: Run OLM Upgrade e2e Ray cluster Job submit test + run: | + export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} + echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV + set -euo pipefail + go test -timeout 30m -v ./test/e2e -run TestMnistJobSubmit -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + env: + RUN_OLM_TESTS: true + - name: Run e2e tests against built operator run: | export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} diff --git a/test/e2e/mnist_rayjob.py b/test/e2e/mnist_rayjob.py new file mode 100644 index 00000000..99cfcb1a --- /dev/null +++ b/test/e2e/mnist_rayjob.py @@ -0,0 +1,44 @@ +import sys + +from time import sleep + +from torchx.specs.api import AppState, is_terminal + +from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration +from codeflare_sdk.job.jobs import DDPJobDefinition + +namespace = sys.argv[1] + +cluster = Cluster(ClusterConfiguration(name='mnist')) + +jobdef = DDPJobDefinition( + name="mnist", + script="mnist.py", + scheduler_args={"requirements": "requirements.txt"}, +) +job = jobdef.submit(cluster) + +done = False +time = 0 +timeout = 300 +while not done: + status = job.status() + if is_terminal(status.state): + break + if not done: + print(status) + if timeout and time >= timeout: + raise TimeoutError(f"job has timed out after waiting {timeout}s") + sleep(5) + time += 5 + +print(f"Job has completed: {status.state}") + +print(job.logs()) + +cluster.down() + +if not status.state == AppState.SUCCEEDED: + exit(1) +else: + exit(0) diff --git a/test/e2e/olm_upgrade_test.go b/test/e2e/olm_upgrade_test.go new file mode 100644 index 00000000..3dde139f --- /dev/null +++ b/test/e2e/olm_upgrade_test.go @@ -0,0 +1,322 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "os" + "testing" + + . "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/project-codeflare/codeflare-operator/test/support" +) + +var namespaceName string = "test-ns-olmupgrade" +var namespace *corev1.Namespace + +// Creates a Ray cluster +func TestMNISTRayClusterUp(t *testing.T) { + + test := With(t) + test.T().Parallel() + if os.Getenv("RUN_OLM_TESTS") == "true" { + // Create a namespace + namespace := CreateTestNamespaceWithName(test, namespaceName) + test.T().Logf("Created namespace %s successfully", namespace.Name) + + // Test configuration + config := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist-raycluster-sdk", + Namespace: namespace.Name, + }, + BinaryData: map[string][]byte{ + // SDK script + "raycluster_sdk.py": ReadFile(test, "raycluster_sdk.py"), + }, + Immutable: Ptr(true), + } + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) + + // SDK client RBAC + serviceAccount := &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk-user", + Namespace: namespace.Name, + }, + } + serviceAccount, err = test.Client().Core().CoreV1().ServiceAccounts(namespace.Name).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + role := &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "Role", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + Namespace: namespace.Name, + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1alpha1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, + }, + }, + } + role, err = test.Client().Core().RbacV1().Roles(namespace.Name).Create(test.Ctx(), role, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + roleBinding := &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: rbacv1.SchemeGroupVersion.String(), + Kind: "RoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "Role", + Name: role.Name, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + APIGroup: corev1.SchemeGroupVersion.Group, + Name: serviceAccount.Name, + Namespace: serviceAccount.Namespace, + }, + }, + } + _, err = test.Client().Core().RbacV1().RoleBindings(namespace.Name).Create(test.Ctx(), roleBinding, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + Namespace: namespace.Name, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + BackoffLimit: Ptr(int32(0)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed + // See https://github.com/project-codeflare/codeflare-sdk/pull/146 + Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python raycluster_sdk.py" + " " + namespace.Name}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + MountPath: "/test", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Name, + }, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: serviceAccount.Name, + }, + }, + }, + } + job, err = test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + + // Retrieving the job logs once it has completed or timed out + defer WriteJobLogs(test, job.Namespace, job.Name) + + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + // Assert the job has completed successfully + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + + } else { + test.T().Skip("Skipping OLM upgarde test because RUN_OLM_TESTS is not set") + } +} + +// Submit a Job to the Ray cluster and trains the MNIST dataset using the CodeFlare SDK. +func TestMnistJobSubmit(t *testing.T) { + + test := With(t) + test.T().Parallel() + if os.Getenv("RUN_OLM_TESTS") == "true" { + + // Test configuration + config := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist-ray-job", + Namespace: namespaceName, + }, + BinaryData: map[string][]byte{ + // SDK script + "mnist_rayjob.py": ReadFile(test, "mnist_rayjob.py"), + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }, + Immutable: Ptr(true), + } + config, err := test.Client().Core().CoreV1().ConfigMaps(namespaceName).Create(test.Ctx(), config, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) + + // SDK client RBAC + serviceAccount := &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ServiceAccount", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ray-user", + Namespace: namespaceName, + }, + } + serviceAccount, err = test.Client().Core().CoreV1().ServiceAccounts(namespaceName).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "rayjob", + Namespace: namespaceName, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + BackoffLimit: Ptr(int32(0)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed + // See https://github.com/project-codeflare/codeflare-sdk/pull/146 + Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_rayjob.py" + " " + namespaceName}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + MountPath: "/test", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Name, + }, + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: serviceAccount.Name, + }, + }, + }, + } + job, err = test.Client().Core().BatchV1().Jobs(namespaceName).Create(test.Ctx(), job, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + + // Retrieving the job logs once it has completed or timed out + defer WriteJobLogs(test, job.Namespace, job.Name) + + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + // Assert the job has completed successfully + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + } else { + test.T().Skip("Skipping OLM upgarde test because RUN_OLM_TESTS is not set") + } + +} diff --git a/test/e2e/raycluster_sdk.py b/test/e2e/raycluster_sdk.py new file mode 100644 index 00000000..5a8abf5c --- /dev/null +++ b/test/e2e/raycluster_sdk.py @@ -0,0 +1,29 @@ +import sys + +from time import sleep + +from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration + +namespace = sys.argv[1] + +cluster = Cluster(ClusterConfiguration( + name='mnist', + namespace=namespace, + num_workers=1, + min_cpus='500m', + max_cpus=1, + min_memory=0.5, + max_memory=1, + num_gpus=0, + instascale=False, +)) + +cluster.up() + +cluster.status() + +cluster.wait_ready() + +cluster.status() + +cluster.details() From 2a367fa9457a895bc7de50c54e99e27858ceec13 Mon Sep 17 00:00:00 2001 From: Srihari Date: Thu, 14 Sep 2023 13:32:41 +0530 Subject: [PATCH 258/377] Create OLM upgrade e2e scenario using codeflare SDK --- test/e2e/olm_upgrade_test.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/e2e/olm_upgrade_test.go b/test/e2e/olm_upgrade_test.go index 3dde139f..acb25a4b 100644 --- a/test/e2e/olm_upgrade_test.go +++ b/test/e2e/olm_upgrade_test.go @@ -42,9 +42,16 @@ func TestMNISTRayClusterUp(t *testing.T) { test.T().Parallel() if os.Getenv("RUN_OLM_TESTS") == "true" { // Create a namespace - namespace := CreateTestNamespaceWithName(test, namespaceName) + namespace = CreateTestNamespaceWithName(test, namespaceName) test.T().Logf("Created namespace %s successfully", namespace.Name) + // Delete namespace only if test failed + defer func() { + if t.Failed() { + DeleteTestNamespace(test, namespaceName) + } + }() + // Test configuration config := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ @@ -212,6 +219,9 @@ func TestMnistJobSubmit(t *testing.T) { test.T().Parallel() if os.Getenv("RUN_OLM_TESTS") == "true" { + //delete the namespace after test complete + defer DeleteTestNamespace(test, namespaceName) + // Test configuration config := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ @@ -315,6 +325,7 @@ func TestMnistJobSubmit(t *testing.T) { // Assert the job has completed successfully test.Expect(GetJob(test, job.Namespace, job.Name)). To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + } else { test.T().Skip("Skipping OLM upgarde test because RUN_OLM_TESTS is not set") } From 2d99cc93bec02b6999e4a773e5fb9da866ed40ab Mon Sep 17 00:00:00 2001 From: Srihari Date: Thu, 23 Nov 2023 12:15:37 +0530 Subject: [PATCH 259/377] rebase and resolving conflicts --- .github/workflows/olm_tests.yaml | 9 +- go.mod | 1 - test/e2e/mnist_rayjob.py | 2 +- test/e2e/olm_upgrade_test.go | 333 ------------------------- test/e2e/raycluster_sdk.py | 29 --- test/e2e/start_ray_cluster.py | 51 ++++ test/upgrade/olm_upgrade_test.go | 403 +++++++++++++++++++++++++++++++ 7 files changed, 458 insertions(+), 370 deletions(-) delete mode 100644 test/e2e/olm_upgrade_test.go delete mode 100644 test/e2e/raycluster_sdk.py create mode 100644 test/e2e/start_ray_cluster.py create mode 100644 test/upgrade/olm_upgrade_test.go diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index de1de85b..927dced7 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -121,14 +121,13 @@ jobs: OPM_BUNDLE_OPT: "--use-http" BUNDLE_PUSH_OPT: "--tls-verify=false" CATALOG_PUSH_OPT: "--tls-verify=false" + - name: Run OLM Upgrade e2e Ray cluster Up test run: | export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV set -euo pipefail - go test -timeout 30m -v ./test/e2e -run TestMNISTRayClusterUp -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - env: - RUN_OLM_TESTS: true + go test -timeout 30m -v ./test/upgrade -run TestMNISTRayClusterUp -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Update Operator to the built version run: | @@ -164,9 +163,7 @@ jobs: export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV set -euo pipefail - go test -timeout 30m -v ./test/e2e -run TestMnistJobSubmit -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - env: - RUN_OLM_TESTS: true + go test -timeout 30m -v ./test/upgrade -run TestMnistJobSubmit -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Run e2e tests against built operator run: | diff --git a/go.mod b/go.mod index e9f358d4..c7405e30 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 - github.com/project-codeflare/instascale v0.3.1 github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 diff --git a/test/e2e/mnist_rayjob.py b/test/e2e/mnist_rayjob.py index 99cfcb1a..7ffb3927 100644 --- a/test/e2e/mnist_rayjob.py +++ b/test/e2e/mnist_rayjob.py @@ -9,7 +9,7 @@ namespace = sys.argv[1] -cluster = Cluster(ClusterConfiguration(name='mnist')) +cluster = Cluster(ClusterConfiguration(name='mnist',namespace=namespace)) jobdef = DDPJobDefinition( name="mnist", diff --git a/test/e2e/olm_upgrade_test.go b/test/e2e/olm_upgrade_test.go deleted file mode 100644 index acb25a4b..00000000 --- a/test/e2e/olm_upgrade_test.go +++ /dev/null @@ -1,333 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "os" - "testing" - - . "github.com/onsi/gomega" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" - - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/project-codeflare/codeflare-operator/test/support" -) - -var namespaceName string = "test-ns-olmupgrade" -var namespace *corev1.Namespace - -// Creates a Ray cluster -func TestMNISTRayClusterUp(t *testing.T) { - - test := With(t) - test.T().Parallel() - if os.Getenv("RUN_OLM_TESTS") == "true" { - // Create a namespace - namespace = CreateTestNamespaceWithName(test, namespaceName) - test.T().Logf("Created namespace %s successfully", namespace.Name) - - // Delete namespace only if test failed - defer func() { - if t.Failed() { - DeleteTestNamespace(test, namespaceName) - } - }() - - // Test configuration - config := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist-raycluster-sdk", - Namespace: namespace.Name, - }, - BinaryData: map[string][]byte{ - // SDK script - "raycluster_sdk.py": ReadFile(test, "raycluster_sdk.py"), - }, - Immutable: Ptr(true), - } - config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - - // SDK client RBAC - serviceAccount := &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ServiceAccount", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk-user", - Namespace: namespace.Name, - }, - } - serviceAccount, err = test.Client().Core().CoreV1().ServiceAccounts(namespace.Name).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - - role := &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{ - APIVersion: rbacv1.SchemeGroupVersion.String(), - Kind: "Role", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", - Namespace: namespace.Name, - }, - Rules: []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1alpha1.GroupVersion.Group}, - Resources: []string{"rayclusters", "rayclusters/status"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"route.openshift.io"}, - Resources: []string{"routes"}, - }, - }, - } - role, err = test.Client().Core().RbacV1().Roles(namespace.Name).Create(test.Ctx(), role, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - - roleBinding := &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: rbacv1.SchemeGroupVersion.String(), - Kind: "RoleBinding", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: rbacv1.SchemeGroupVersion.Group, - Kind: "Role", - Name: role.Name, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - APIGroup: corev1.SchemeGroupVersion.Group, - Name: serviceAccount.Name, - Namespace: serviceAccount.Namespace, - }, - }, - } - _, err = test.Client().Core().RbacV1().RoleBindings(namespace.Name).Create(test.Ctx(), roleBinding, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", - Namespace: namespace.Name, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - BackoffLimit: Ptr(int32(0)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed - // See https://github.com/project-codeflare/codeflare-sdk/pull/146 - Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", - Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python raycluster_sdk.py" + " " + namespace.Name}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: serviceAccount.Name, - }, - }, - }, - } - job, err = test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) - - // Retrieving the job logs once it has completed or timed out - defer WriteJobLogs(test, job.Namespace, job.Name) - - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( - Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), - )) - - // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) - - } else { - test.T().Skip("Skipping OLM upgarde test because RUN_OLM_TESTS is not set") - } -} - -// Submit a Job to the Ray cluster and trains the MNIST dataset using the CodeFlare SDK. -func TestMnistJobSubmit(t *testing.T) { - - test := With(t) - test.T().Parallel() - if os.Getenv("RUN_OLM_TESTS") == "true" { - - //delete the namespace after test complete - defer DeleteTestNamespace(test, namespaceName) - - // Test configuration - config := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist-ray-job", - Namespace: namespaceName, - }, - BinaryData: map[string][]byte{ - // SDK script - "mnist_rayjob.py": ReadFile(test, "mnist_rayjob.py"), - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }, - Immutable: Ptr(true), - } - config, err := test.Client().Core().CoreV1().ConfigMaps(namespaceName).Create(test.Ctx(), config, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - - // SDK client RBAC - serviceAccount := &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ServiceAccount", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "ray-user", - Namespace: namespaceName, - }, - } - serviceAccount, err = test.Client().Core().CoreV1().ServiceAccounts(namespaceName).Create(test.Ctx(), serviceAccount, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "rayjob", - Namespace: namespaceName, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - BackoffLimit: Ptr(int32(0)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed - // See https://github.com/project-codeflare/codeflare-sdk/pull/146 - Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", - Command: []string{"/bin/sh", "-c", "pip install codeflare-sdk==" + GetCodeFlareSDKVersion() + " && cp /test/* . && python mnist_rayjob.py" + " " + namespaceName}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: serviceAccount.Name, - }, - }, - }, - } - job, err = test.Client().Core().BatchV1().Jobs(namespaceName).Create(test.Ctx(), job, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) - - // Retrieving the job logs once it has completed or timed out - defer WriteJobLogs(test, job.Namespace, job.Name) - - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( - Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), - )) - - // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) - - } else { - test.T().Skip("Skipping OLM upgarde test because RUN_OLM_TESTS is not set") - } - -} diff --git a/test/e2e/raycluster_sdk.py b/test/e2e/raycluster_sdk.py deleted file mode 100644 index 5a8abf5c..00000000 --- a/test/e2e/raycluster_sdk.py +++ /dev/null @@ -1,29 +0,0 @@ -import sys - -from time import sleep - -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration - -namespace = sys.argv[1] - -cluster = Cluster(ClusterConfiguration( - name='mnist', - namespace=namespace, - num_workers=1, - min_cpus='500m', - max_cpus=1, - min_memory=0.5, - max_memory=1, - num_gpus=0, - instascale=False, -)) - -cluster.up() - -cluster.status() - -cluster.wait_ready() - -cluster.status() - -cluster.details() diff --git a/test/e2e/start_ray_cluster.py b/test/e2e/start_ray_cluster.py new file mode 100644 index 00000000..238bccdf --- /dev/null +++ b/test/e2e/start_ray_cluster.py @@ -0,0 +1,51 @@ +import sys +import os + +from time import sleep + +from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration + +namespace = sys.argv[1] +ray_image = os.getenv('RAY_IMAGE') +host = os.getenv('CLUSTER_HOSTNAME') + +ingress_options = {} +if host is not None: + ingress_options = { + "ingresses": [ + { + "ingressName": "ray-dashboard", + "port": 8265, + "pathType": "Prefix", + "path": "/", + "host": host, + }, + ] + } + + +cluster = Cluster(ClusterConfiguration( + name='mnist', + namespace=namespace, + num_workers=1, + head_cpus='500m', + head_memory=2, + min_cpus='500m', + max_cpus=1, + min_memory=0.5, + max_memory=2, + num_gpus=0, + instascale=False, + image=ray_image, + ingress_options=ingress_options, +)) + +cluster.up() + +cluster.status() + +cluster.wait_ready() + +cluster.status() + +cluster.details() diff --git a/test/upgrade/olm_upgrade_test.go b/test/upgrade/olm_upgrade_test.go new file mode 100644 index 00000000..0d27c064 --- /dev/null +++ b/test/upgrade/olm_upgrade_test.go @@ -0,0 +1,403 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "testing" + + . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/project-codeflare/codeflare-operator/test/e2e" +) + +var ( + namespaceName = "test-ns-olmupgrade" +) + +// Creates a Ray cluster +func TestMNISTRayClusterUp(t *testing.T) { + + test := With(t) + test.T().Parallel() + + // Create a namespace + namespace := CreateTestNamespaceWithName(test, namespaceName) + test.T().Logf("Created namespace %s successfully", namespace.Name) + + // Delete namespace only if test failed + defer func() { + if t.Failed() { + DeleteTestNamespace(test, namespace) + } else { + StoreNamespaceLogs(test, namespace) + } + }() + + // Test configuration + config := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // SDK script + "start_ray_cluster.py": ReadFile(test, "start_ray_cluster.py"), + // codeflare-sdk installation script + "install-codeflare-sdk.sh": ReadFile(test, "install-codeflare-sdk.sh"), + }) + + // Create RBAC, retrieve token for user with limited rights + policyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"ingresses"}, + }, + } + + // Create cluster wide RBAC, required for SDK OpenShift check + // TODO reevaluate once SDK change OpenShift detection logic + clusterPolicyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"config.openshift.io"}, + Resources: []string{"ingresses"}, + ResourceNames: []string{"cluster"}, + }, + } + + sa := CreateServiceAccount(test, namespace.Name) + role := CreateRole(test, namespace.Name, policyRules) + CreateRoleBinding(test, namespace.Name, sa, role) + clusterRole := CreateClusterRole(test, clusterPolicyRules) + CreateClusterRoleBinding(test, sa, clusterRole) + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "sdk", + Namespace: namespace.Name, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + BackoffLimit: Ptr(int32(0)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed + // See https://github.com/project-codeflare/codeflare-sdk/pull/146 + Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Env: []corev1.EnvVar{ + {Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "RAY_IMAGE", Value: GetRayImage()}, + }, + Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python start_ray_cluster.py" + " " + namespace.Name}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + MountPath: "/test", + }, + { + Name: "codeflare-sdk", + MountPath: "/codeflare-sdk", + }, + { + Name: "workdir", + MountPath: "/workdir", + }, + }, + WorkingDir: "/workdir", + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: Ptr(false), + SeccompProfile: &corev1.SeccompProfile{ + Type: "RuntimeDefault", + }, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + RunAsNonRoot: Ptr(true), + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Name, + }, + }, + }, + }, + { + Name: "codeflare-sdk", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "workdir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: sa.Name, + }, + }, + }, + } + if GetClusterType(test) == KindCluster { + // Take first KinD node and redirect pod hostname requests there + node := GetNodes(test)[0] + hostname := GetClusterHostname(test) + IP := GetNodeInternalIP(test, node) + + test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP) + job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{ + { + IP: IP, + Hostnames: []string{hostname}, + }, + } + + // Propagate hostname into Python code as env variable + hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname} + job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar) + } + + job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + // Assert the job has completed successfully + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) +} + +// Submit a Job to the Ray cluster and trains the MNIST dataset using the CodeFlare SDK. +func TestMnistJobSubmit(t *testing.T) { + + test := With(t) + test.T().Parallel() + + namespace := GetNamespaceWithName(test, namespaceName) + + //delete the namespace after test complete + defer DeleteTestNamespace(test, namespace) + + // Test configuration + config := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // SDK script + "mnist_rayjob.py": ReadFile(test, "mnist_rayjob.py"), + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + // codeflare-sdk installation script + "install-codeflare-sdk.sh": ReadFile(test, "install-codeflare-sdk.sh"), + }) + + // Create RBAC, retrieve token for user with limited rights + policyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, + APIGroups: []string{mcadv1beta1.GroupName}, + Resources: []string{"appwrappers"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{rayv1.GroupVersion.Group}, + Resources: []string{"rayclusters", "rayclusters/status"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"route.openshift.io"}, + Resources: []string{"routes"}, + }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"ingresses"}, + }, + } + + // Create cluster wide RBAC, required for SDK OpenShift check + // TODO reevaluate once SDK change OpenShift detection logic + clusterPolicyRules := []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"config.openshift.io"}, + Resources: []string{"ingresses"}, + ResourceNames: []string{"cluster"}, + }, + } + + serviceAccount := CreateServiceAccount(test, namespace.Name) + role := CreateRole(test, namespace.Name, policyRules) + CreateRoleBinding(test, namespace.Name, serviceAccount, role) + clusterRole := CreateClusterRole(test, clusterPolicyRules) + CreateClusterRoleBinding(test, serviceAccount, clusterRole) + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + APIVersion: batchv1.SchemeGroupVersion.String(), + Kind: "Job", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "rayjob", + Namespace: namespaceName, + }, + Spec: batchv1.JobSpec{ + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), + BackoffLimit: Ptr(int32(0)), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed + // See https://github.com/project-codeflare/codeflare-sdk/pull/146 + Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python mnist_rayjob.py" + " " + namespaceName}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "test", + MountPath: "/test", + }, + { + Name: "codeflare-sdk", + MountPath: "/codeflare-sdk", + }, + { + Name: "workdir", + MountPath: "/workdir", + }, + }, + WorkingDir: "/workdir", + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: Ptr(false), + SeccompProfile: &corev1.SeccompProfile{ + Type: "RuntimeDefault", + }, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + RunAsNonRoot: Ptr(true), + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "test", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: config.Name, + }, + }, + }, + }, + { + Name: "codeflare-sdk", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "workdir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: serviceAccount.Name, + }, + }, + }, + } + + if GetClusterType(test) == KindCluster { + // Take first KinD node and redirect pod hostname requests there + node := GetNodes(test)[0] + hostname := GetClusterHostname(test) + IP := GetNodeInternalIP(test, node) + + test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP) + job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{ + { + IP: IP, + Hostnames: []string{hostname}, + }, + } + + // Propagate hostname into Python code as env variable + hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname} + job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar) + } + + job, err := test.Client().Core().BatchV1().Jobs(namespaceName).Create(test.Ctx(), job, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + + test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + // Assert the job has completed successfully + test.Expect(GetJob(test, job.Namespace, job.Name)). + To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + +} From 0afa252a9a4178cd9e75f1e87e12c6954f41c105 Mon Sep 17 00:00:00 2001 From: Srihari Date: Mon, 4 Dec 2023 16:42:56 +0530 Subject: [PATCH 260/377] Create e2e scenario covering upgrade during training --- .github/workflows/olm_tests.yaml | 8 +- go.mod | 1 + test/e2e/mnist_rayjob.py | 44 ---- test/e2e/start_ray_cluster.py | 51 ----- test/upgrade/olm_upgrade_test.go | 345 +++++++------------------------ 5 files changed, 79 insertions(+), 370 deletions(-) delete mode 100644 test/e2e/mnist_rayjob.py delete mode 100644 test/e2e/start_ray_cluster.py diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 927dced7..1dcb6f2d 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -122,12 +122,12 @@ jobs: BUNDLE_PUSH_OPT: "--tls-verify=false" CATALOG_PUSH_OPT: "--tls-verify=false" - - name: Run OLM Upgrade e2e Ray cluster Up test + - name: Run OLM Upgrade e2e AppWrapper creation test run: | export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV set -euo pipefail - go test -timeout 30m -v ./test/upgrade -run TestMNISTRayClusterUp -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + go test -timeout 30m -v ./test/upgrade -run TestMNISTCreateAppWrapper -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Update Operator to the built version run: | @@ -158,12 +158,12 @@ jobs: SUBSCRIPTION_NAME: "codeflare-operator" SUBSCRIPTION_NAMESPACE: "openshift-operators" - - name: Run OLM Upgrade e2e Ray cluster Job submit test + - name: Run OLM Upgrade e2e Appwrapper Job status test to monitor training run: | export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV set -euo pipefail - go test -timeout 30m -v ./test/upgrade -run TestMnistJobSubmit -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + go test -timeout 30m -v ./test/upgrade -run TestMNISTCheckAppWrapperStatus -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - name: Run e2e tests against built operator run: | diff --git a/go.mod b/go.mod index c7405e30..e9f358d4 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 + github.com/project-codeflare/instascale v0.3.1 github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 diff --git a/test/e2e/mnist_rayjob.py b/test/e2e/mnist_rayjob.py deleted file mode 100644 index 7ffb3927..00000000 --- a/test/e2e/mnist_rayjob.py +++ /dev/null @@ -1,44 +0,0 @@ -import sys - -from time import sleep - -from torchx.specs.api import AppState, is_terminal - -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration -from codeflare_sdk.job.jobs import DDPJobDefinition - -namespace = sys.argv[1] - -cluster = Cluster(ClusterConfiguration(name='mnist',namespace=namespace)) - -jobdef = DDPJobDefinition( - name="mnist", - script="mnist.py", - scheduler_args={"requirements": "requirements.txt"}, -) -job = jobdef.submit(cluster) - -done = False -time = 0 -timeout = 300 -while not done: - status = job.status() - if is_terminal(status.state): - break - if not done: - print(status) - if timeout and time >= timeout: - raise TimeoutError(f"job has timed out after waiting {timeout}s") - sleep(5) - time += 5 - -print(f"Job has completed: {status.state}") - -print(job.logs()) - -cluster.down() - -if not status.state == AppState.SUCCEEDED: - exit(1) -else: - exit(0) diff --git a/test/e2e/start_ray_cluster.py b/test/e2e/start_ray_cluster.py deleted file mode 100644 index 238bccdf..00000000 --- a/test/e2e/start_ray_cluster.py +++ /dev/null @@ -1,51 +0,0 @@ -import sys -import os - -from time import sleep - -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration - -namespace = sys.argv[1] -ray_image = os.getenv('RAY_IMAGE') -host = os.getenv('CLUSTER_HOSTNAME') - -ingress_options = {} -if host is not None: - ingress_options = { - "ingresses": [ - { - "ingressName": "ray-dashboard", - "port": 8265, - "pathType": "Prefix", - "path": "/", - "host": host, - }, - ] - } - - -cluster = Cluster(ClusterConfiguration( - name='mnist', - namespace=namespace, - num_workers=1, - head_cpus='500m', - head_memory=2, - min_cpus='500m', - max_cpus=1, - min_memory=0.5, - max_memory=2, - num_gpus=0, - instascale=False, - image=ray_image, - ingress_options=ingress_options, -)) - -cluster.up() - -cluster.status() - -cluster.wait_ready() - -cluster.status() - -cluster.details() diff --git a/test/upgrade/olm_upgrade_test.go b/test/upgrade/olm_upgrade_test.go index 0d27c064..2abd086e 100644 --- a/test/upgrade/olm_upgrade_test.go +++ b/test/upgrade/olm_upgrade_test.go @@ -22,29 +22,27 @@ import ( . "github.com/onsi/gomega" . "github.com/project-codeflare/codeflare-common/support" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/project-codeflare/codeflare-operator/test/e2e" ) var ( - namespaceName = "test-ns-olmupgrade" + namespaceName = "test-ns-olmupgrade" + appwrapperName = "mnist" + jobName = "mnist-job" ) -// Creates a Ray cluster -func TestMNISTRayClusterUp(t *testing.T) { - +func TestMNISTCreateAppWrapper(t *testing.T) { test := With(t) test.T().Parallel() // Create a namespace namespace := CreateTestNamespaceWithName(test, namespaceName) - test.T().Logf("Created namespace %s successfully", namespace.Name) // Delete namespace only if test failed defer func() { @@ -56,105 +54,61 @@ func TestMNISTRayClusterUp(t *testing.T) { }() // Test configuration - config := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // SDK script - "start_ray_cluster.py": ReadFile(test, "start_ray_cluster.py"), - // codeflare-sdk installation script - "install-codeflare-sdk.sh": ReadFile(test, "install-codeflare-sdk.sh"), - }) - - // Create RBAC, retrieve token for user with limited rights - policyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1.GroupVersion.Group}, - Resources: []string{"rayclusters", "rayclusters/status"}, + config := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"route.openshift.io"}, - Resources: []string{"routes"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "mnist-mcad", + Namespace: namespace.Name, }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"ingresses"}, + BinaryData: map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), }, + Immutable: Ptr(true), } + config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - // Create cluster wide RBAC, required for SDK OpenShift check - // TODO reevaluate once SDK change OpenShift detection logic - clusterPolicyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"config.openshift.io"}, - Resources: []string{"ingresses"}, - ResourceNames: []string{"cluster"}, - }, - } - - sa := CreateServiceAccount(test, namespace.Name) - role := CreateRole(test, namespace.Name, policyRules) - CreateRoleBinding(test, namespace.Name, sa, role) - clusterRole := CreateClusterRole(test, clusterPolicyRules) - CreateClusterRoleBinding(test, sa, clusterRole) - + // Batch Job job := &batchv1.Job{ TypeMeta: metav1.TypeMeta{ APIVersion: batchv1.SchemeGroupVersion.String(), Kind: "Job", }, ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", + Name: jobName, Namespace: namespace.Name, }, Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - BackoffLimit: Ptr(int32(0)), + Completions: Ptr(int32(1)), + Parallelism: Ptr(int32(1)), Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "test", - // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed - // See https://github.com/project-codeflare/codeflare-sdk/pull/146 - Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", + Name: "job", + Image: GetPyTorchImage(), Env: []corev1.EnvVar{ - {Name: "PYTHONUSERBASE", Value: "/workdir"}, - {Name: "RAY_IMAGE", Value: GetRayImage()}, + corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"}, }, - Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python start_ray_cluster.py" + " " + namespace.Name}, + Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ { Name: "test", MountPath: "/test", }, - { - Name: "codeflare-sdk", - MountPath: "/codeflare-sdk", - }, { Name: "workdir", MountPath: "/workdir", }, }, WorkingDir: "/workdir", - SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: Ptr(false), - SeccompProfile: &corev1.SeccompProfile{ - Type: "RuntimeDefault", - }, - Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - RunAsNonRoot: Ptr(true), - }, }, }, Volumes: []corev1.Volume{ @@ -168,12 +122,6 @@ func TestMNISTRayClusterUp(t *testing.T) { }, }, }, - { - Name: "codeflare-sdk", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, { Name: "workdir", VolumeSource: corev1.VolumeSource{ @@ -181,223 +129,78 @@ func TestMNISTRayClusterUp(t *testing.T) { }, }, }, - RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: sa.Name, + RestartPolicy: corev1.RestartPolicyNever, }, }, }, } - if GetClusterType(test) == KindCluster { - // Take first KinD node and redirect pod hostname requests there - node := GetNodes(test)[0] - hostname := GetClusterHostname(test) - IP := GetNodeInternalIP(test, node) - test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP) - job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{ - { - IP: IP, - Hostnames: []string{hostname}, + // Create an AppWrapper resource + aw := &mcadv1beta1.AppWrapper{ + ObjectMeta: metav1.ObjectMeta{ + Name: appwrapperName, + Namespace: namespace.Name, + }, + Spec: mcadv1beta1.AppWrapperSpec{ + AggrResources: mcadv1beta1.AppWrapperResourceList{ + GenericItems: []mcadv1beta1.AppWrapperGenericResource{ + { + DesiredAvailable: 1, + CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ + { + Replicas: 1, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("250m"), + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }, + }, + GenericTemplate: Raw(test, job), + CompletionStatus: "Complete", + }, + }, }, - } - - // Propagate hostname into Python code as env variable - hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname} - job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar) + }, } - job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) + _, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + test.T().Logf("Created MCAD AppWrapper %s/%s successfully", aw.Namespace, aw.Name) - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( - Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), - )) + test.T().Logf("Waiting for MCAD %s/%s to be running", aw.Namespace, aw.Name) + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) } -// Submit a Job to the Ray cluster and trains the MNIST dataset using the CodeFlare SDK. -func TestMnistJobSubmit(t *testing.T) { - +func TestMNISTCheckAppWrapperStatus(t *testing.T) { test := With(t) test.T().Parallel() + // get namespace namespace := GetNamespaceWithName(test, namespaceName) //delete the namespace after test complete defer DeleteTestNamespace(test, namespace) - // Test configuration - config := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // SDK script - "mnist_rayjob.py": ReadFile(test, "mnist_rayjob.py"), - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - // codeflare-sdk installation script - "install-codeflare-sdk.sh": ReadFile(test, "install-codeflare-sdk.sh"), - }) - - // Create RBAC, retrieve token for user with limited rights - policyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1.GroupVersion.Group}, - Resources: []string{"rayclusters", "rayclusters/status"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"route.openshift.io"}, - Resources: []string{"routes"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"ingresses"}, - }, - } - - // Create cluster wide RBAC, required for SDK OpenShift check - // TODO reevaluate once SDK change OpenShift detection logic - clusterPolicyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"config.openshift.io"}, - Resources: []string{"ingresses"}, - ResourceNames: []string{"cluster"}, - }, - } - - serviceAccount := CreateServiceAccount(test, namespace.Name) - role := CreateRole(test, namespace.Name, policyRules) - CreateRoleBinding(test, namespace.Name, serviceAccount, role) - clusterRole := CreateClusterRole(test, clusterPolicyRules) - CreateClusterRoleBinding(test, serviceAccount, clusterRole) - - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "rayjob", - Namespace: namespaceName, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - BackoffLimit: Ptr(int32(0)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed - // See https://github.com/project-codeflare/codeflare-sdk/pull/146 - Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", - Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python mnist_rayjob.py" + " " + namespaceName}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - { - Name: "codeflare-sdk", - MountPath: "/codeflare-sdk", - }, - { - Name: "workdir", - MountPath: "/workdir", - }, - }, - WorkingDir: "/workdir", - SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: Ptr(false), - SeccompProfile: &corev1.SeccompProfile{ - Type: "RuntimeDefault", - }, - Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - RunAsNonRoot: Ptr(true), - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - { - Name: "codeflare-sdk", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "workdir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: serviceAccount.Name, - }, - }, - }, - } - - if GetClusterType(test) == KindCluster { - // Take first KinD node and redirect pod hostname requests there - node := GetNodes(test)[0] - hostname := GetClusterHostname(test) - IP := GetNodeInternalIP(test, node) - - test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP) - job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{ - { - IP: IP, - Hostnames: []string{hostname}, - }, - } - - // Propagate hostname into Python code as env variable - hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname} - job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar) - } - - job, err := test.Client().Core().BatchV1().Jobs(namespaceName).Create(test.Ctx(), job, metav1.CreateOptions{}) + aw, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Get(test.Ctx(), appwrapperName, metav1.GetOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) + job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Get(test.Ctx(), jobName, metav1.GetOptions{}) + test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong).Should( Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)), + WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateFailed)), )) // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) + test.Expect(GetAppWrapper(test, namespace, aw.Name)). + To(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) } From a4590eaa09d90957d1ad26e7e9ef3479ab63a772 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 12 Dec 2023 11:13:18 +0100 Subject: [PATCH 261/377] Adjust upgrade e2e test to pause Job before upgrade --- test/upgrade/olm_upgrade_test.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/test/upgrade/olm_upgrade_test.go b/test/upgrade/olm_upgrade_test.go index 2abd086e..ac0783e0 100644 --- a/test/upgrade/olm_upgrade_test.go +++ b/test/upgrade/olm_upgrade_test.go @@ -27,13 +27,14 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" . "github.com/project-codeflare/codeflare-operator/test/e2e" ) var ( namespaceName = "test-ns-olmupgrade" - appwrapperName = "mnist" + appWrapperName = "mnist" jobName = "mnist-job" ) @@ -132,13 +133,14 @@ func TestMNISTCreateAppWrapper(t *testing.T) { RestartPolicy: corev1.RestartPolicyNever, }, }, + Suspend: Ptr(true), }, } // Create an AppWrapper resource aw := &mcadv1beta1.AppWrapper{ ObjectMeta: metav1.ObjectMeta{ - Name: appwrapperName, + Name: appWrapperName, Namespace: namespace.Name, }, Spec: mcadv1beta1.AppWrapperSpec{ @@ -187,20 +189,27 @@ func TestMNISTCheckAppWrapperStatus(t *testing.T) { //delete the namespace after test complete defer DeleteTestNamespace(test, namespace) - aw, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Get(test.Ctx(), appwrapperName, metav1.GetOptions{}) + // Patch job to resume execution + patch := []byte(`[{"op":"replace","path":"/spec/suspend","value": false}]`) + job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Patch(test.Ctx(), jobName, types.JSONPatchType, patch, metav1.PatchOptions{}) test.Expect(err).NotTo(HaveOccurred()) - job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Get(test.Ctx(), jobName, metav1.GetOptions{}) - test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong).Should( + test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( + Or( + WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), + WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), + )) + + test.T().Logf("Waiting for AppWrapper %s/%s to complete", namespace.Name, appWrapperName) + test.Eventually(AppWrapper(test, namespace, appWrapperName), TestTimeoutShort).Should( Or( WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)), WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateFailed)), )) - // Assert the job has completed successfully - test.Expect(GetAppWrapper(test, namespace, aw.Name)). + // Assert the AppWrapper has completed successfully + test.Expect(GetAppWrapper(test, namespace, appWrapperName)). To(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) } From ce7710d9772b201bb8868c06769ccc21755386c1 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Wed, 13 Dec 2023 17:40:21 +0000 Subject: [PATCH 262/377] Upgrade to go 1.20 and update dependencies + point to a compatible MCAD --- .github/workflows/e2e_tests.yaml | 2 +- .github/workflows/olm_tests.yaml | 2 +- .github/workflows/operator-image.yml | 2 +- .github/workflows/precommit.yml | 2 +- .github/workflows/tag-and-build.yml | 2 +- .github/workflows/unit_tests.yml | 2 +- .github/workflows/verify_generated_files.yml | 4 +- CONTRIBUTING.md | 2 +- Dockerfile | 2 +- Makefile | 2 +- config/manager/manager.yaml | 2 +- go.mod | 50 +++---- go.sum | 137 +++++++++---------- 13 files changed, 100 insertions(+), 111 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 1962a8b8..180bc23c 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -45,7 +45,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 1dcb6f2d..e4716bd0 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -44,7 +44,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 diff --git a/.github/workflows/operator-image.yml b/.github/workflows/operator-image.yml index d2df8683..0c155be1 100644 --- a/.github/workflows/operator-image.yml +++ b/.github/workflows/operator-image.yml @@ -22,7 +22,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Login to Quay.io uses: redhat-actions/podman-login@v1 diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 7ef3a5e4..13c4c7fc 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -26,7 +26,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Activate cache uses: actions/cache@v3 diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 28d27c99..038b77b0 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -64,7 +64,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Verify that release doesn't exist yet shell: bash {0} diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7615ff77..39ab0ef0 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -27,7 +27,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Activate cache uses: actions/cache@v3 diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml index b99a9a06..e2fa269c 100644 --- a/.github/workflows/verify_generated_files.yml +++ b/.github/workflows/verify_generated_files.yml @@ -24,7 +24,7 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Verify that imports are organized run: make verify-imports @@ -35,6 +35,6 @@ jobs: - name: Set Go uses: actions/setup-go@v3 with: - go-version: v1.19 + go-version: v1.20 - name: Verify that the latest WebhookConfigurations, ClusterRoles, and CustomResourceDefinitions have been generated run: make manifests && git diff --exit-code diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9fd4a3f8..8613b92f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Here are a few things to go over before getting started with CodeFlare Operator ## Environment setup The following should be installed in your working environment: - - Go 1.19.x + - Go 1.20.x - [Download release](https://go.dev/dl/) - [Install Instructions](https://go.dev/doc/install) - [Operator SDK](https://sdk.operatorframework.io/docs/installation/) diff --git a/Dockerfile b/Dockerfile index ea02d1b1..f5c51c3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.access.redhat.com/ubi8/go-toolset:1.19.13 as builder +FROM registry.access.redhat.com/ubi8/go-toolset:1.20.10 as builder WORKDIR /workspace # Copy the Go Modules manifests diff --git a/Makefile b/Makefile index d32fdc01..3f0d0f52 100644 --- a/Makefile +++ b/Makefile @@ -229,7 +229,7 @@ SED ?= /usr/bin/sed KUSTOMIZE_VERSION ?= v4.5.4 CODEGEN_VERSION ?= v0.27.2 CONTROLLER_TOOLS_VERSION ?= v0.9.2 -YQ_VERSION ?= v4.30.8 ## latest version that works with go1.19 +YQ_VERSION ?= v4.35.2 ## latest version that works with go1.20 OPERATOR_SDK_VERSION ?= v1.27.0 GH_CLI_VERSION ?= 2.30.0 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ae13f8af..8a4f6451 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -28,7 +28,7 @@ spec: # it is recommended to ensure that all your Pods/Containers are restrictive. # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted # Please uncomment the following code if your project does NOT have to work on old Kubernetes - # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). + # versions < 1.20 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). # seccompProfile: # type: RuntimeDefault containers: diff --git a/go.mod b/go.mod index e9f358d4..ec0700b4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/project-codeflare/codeflare-operator -go 1.19 +go 1.20 require ( github.com/onsi/gomega v1.27.10 @@ -10,28 +10,31 @@ require ( github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 - k8s.io/api v0.26.3 - k8s.io/apimachinery v0.26.3 - k8s.io/client-go v0.26.3 - k8s.io/component-base v0.26.2 + k8s.io/api v0.27.8 + k8s.io/apimachinery v0.27.8 + k8s.io/client-go v0.27.8 + k8s.io/component-base v0.27.8 k8s.io/klog/v2 v2.90.1 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 - sigs.k8s.io/controller-runtime v0.14.6 + sigs.k8s.io/controller-runtime v0.15.3 sigs.k8s.io/yaml v1.3.0 ) replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 +replace github.com/project-codeflare/multi-cluster-app-dispatcher => github.com/ChristianZaccaria/multi-cluster-app-dispatcher v0.0.0-20231213170206-696368415a5a + require ( github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect + github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/coreos/go-systemd/v22 v22.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/eapache/go-resiliency v1.3.0 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect @@ -41,7 +44,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-logr/zapr v1.2.3 // indirect + github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -50,7 +53,7 @@ require ( github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/cel-go v0.12.6 // indirect + github.com/google/cel-go v0.12.7 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -65,24 +68,25 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/microcosm-cc/bluemonday v1.0.18 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/client_golang v1.15.1 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect - go.etcd.io/etcd/api/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect - go.etcd.io/etcd/client/v3 v3.5.5 // indirect + go.etcd.io/etcd/api/v3 v3.5.7 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect + go.etcd.io/etcd/client/v3 v3.5.7 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect go.opentelemetry.io/otel v1.19.0 // indirect @@ -101,7 +105,7 @@ require ( golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect @@ -112,13 +116,13 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.26.1 // indirect - k8s.io/apiserver v0.26.2 // indirect + k8s.io/apiextensions-apiserver v0.27.7 // indirect + k8s.io/apiserver v0.27.8 // indirect k8s.io/klog v1.0.0 // indirect - k8s.io/kms v0.26.2 // indirect - k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d // indirect + k8s.io/kms v0.27.8 // indirect + k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect k8s.io/metrics v0.26.2 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect sigs.k8s.io/custom-metrics-apiserver v0.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect diff --git a/go.sum b/go.sum index 9af7ce38..81c66b13 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ChristianZaccaria/multi-cluster-app-dispatcher v0.0.0-20231213170206-696368415a5a h1:11bIdGaYS1YOxNYyvR8S9VNNhcMDMvjQxdJ4VMEp810= +github.com/ChristianZaccaria/multi-cluster-app-dispatcher v0.0.0-20231213170206-696368415a5a/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= @@ -49,6 +51,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -75,15 +79,14 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.4.0 h1:y9YHcjnjynCd/DVbg5j9L/33jQM3MxJlbj/zWskzfGU= +github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -92,7 +95,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= @@ -101,12 +103,9 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= @@ -114,7 +113,6 @@ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2Vvl github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -126,11 +124,9 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -138,8 +134,8 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= -github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= @@ -194,8 +190,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/cel-go v0.12.6 h1:kjeKudqV0OygrAqA9fX6J55S8gj+Jre2tckIm5RoG4M= -github.com/google/cel-go v0.12.6/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= +github.com/google/cel-go v0.12.7 h1:jM6p55R0MKBg79hZjn1zs2OlrywZ1Vk00rxVvad1/O0= +github.com/google/cel-go v0.12.7/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -348,10 +344,12 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -395,40 +393,37 @@ github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= github.com/project-codeflare/instascale v0.3.1 h1:LIKo5NaX7kDPmAYy1aAkF0yykql3ZdCiiLsBOC4HRcM= github.com/project-codeflare/instascale v0.3.1/go.mod h1:Fdy3daVhz3BHFvPyr9zfH6uFF2yww73U41Wq2S2zDI8= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 h1:6ILHYAFxDkAnQu3CJebGQPQGcmcy7/E/AhRiea6yOTc= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 h1:aoOajXNG2QnTPmnSzbCXLFBXrVk7bzQGrwwVzTXSOzo= github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -440,7 +435,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= @@ -463,7 +458,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -476,16 +471,16 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= -go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= -go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= -go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= -go.etcd.io/etcd/client/v2 v2.305.5 h1:DktRP60//JJpnPC0VBymAN/7V71GHMdjDCBt4ZPXDjI= -go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= -go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= -go.etcd.io/etcd/pkg/v3 v3.5.5 h1:Ablg7T7OkR+AeeeU32kdVhw/AGDsitkKPl7aW73ssjU= -go.etcd.io/etcd/raft/v3 v3.5.5 h1:Ibz6XyZ60OYyRopu73lLM/P+qco3YtlZMOhnXNS051I= -go.etcd.io/etcd/server/v3 v3.5.5 h1:jNjYm/9s+f9A9r6+SC4RvNaz6AqixpOvhrFdT0PvIj0= +go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY= +go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= +go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg= +go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY= +go.etcd.io/etcd/client/v2 v2.305.7 h1:AELPkjNR3/igjbO7CjyF1fPuVPjrblliiKj+Y6xSGOU= +go.etcd.io/etcd/client/v3 v3.5.7 h1:u/OhpiuCgYY8awOHlhIhmGIGpxfBU/GZBUP3m/3/Iz4= +go.etcd.io/etcd/client/v3 v3.5.7/go.mod h1:sOWmj9DZUMyAngS7QQwCyAXXAL6WhgTOPLNS/NabQgw= +go.etcd.io/etcd/pkg/v3 v3.5.7 h1:obOzeVwerFwZ9trMWapU/VjDcYUJb5OfgC1zqEGWO/0= +go.etcd.io/etcd/raft/v3 v3.5.7 h1:aN79qxLmV3SvIq84aNTliYGmjwsW6NqJSnqmI1HLJKc= +go.etcd.io/etcd/server/v3 v3.5.7 h1:BTBD8IJUV7YFgsczZMHhMTS67XuA4KpRquL0MFOJGRk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -515,7 +510,7 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -527,8 +522,7 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -568,7 +562,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -619,7 +612,6 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= @@ -630,7 +622,6 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -691,7 +682,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -745,7 +735,6 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -775,17 +764,17 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= +gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -840,7 +829,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= @@ -862,9 +850,7 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -918,26 +904,26 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.26.3 h1:emf74GIQMTik01Aum9dPP0gAypL8JTLl/lHa4V9RFSU= -k8s.io/api v0.26.3/go.mod h1:PXsqwPMXBSBcL1lJ9CYDKy7kIReUydukS5JiRlxC3qE= -k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI= -k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM= -k8s.io/apimachinery v0.26.3 h1:dQx6PNETJ7nODU3XPtrwkfuubs6w7sX0M8n61zHIV/k= -k8s.io/apimachinery v0.26.3/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= -k8s.io/apiserver v0.26.2 h1:Pk8lmX4G14hYqJd1poHGC08G03nIHVqdJMR0SD3IH3o= -k8s.io/apiserver v0.26.2/go.mod h1:GHcozwXgXsPuOJ28EnQ/jXEM9QeG6HT22YxSNmpYNh8= -k8s.io/client-go v0.26.3 h1:k1UY+KXfkxV2ScEL3gilKcF7761xkYsSD6BC9szIu8s= -k8s.io/client-go v0.26.3/go.mod h1:ZPNu9lm8/dbRIPAgteN30RSXea6vrCpFvq+MateTUuQ= -k8s.io/component-base v0.26.2 h1:IfWgCGUDzrD6wLLgXEstJKYZKAFS2kO+rBRi0p3LqcI= -k8s.io/component-base v0.26.2/go.mod h1:DxbuIe9M3IZPRxPIzhch2m1eT7uFrSBJUBuVCQEBivs= +k8s.io/api v0.27.8 h1:Ja93gbyII5c3TJzWefEwGhlqC5SZksWhzRS+OYHIocU= +k8s.io/api v0.27.8/go.mod h1:2HuWJC6gpx4UScY+ezziNzv6j6Jqd2q0rGgobYSSjcs= +k8s.io/apiextensions-apiserver v0.27.7 h1:YqIOwZAUokzxJIjunmUd4zS1v3JhK34EPXn+pP0/bsU= +k8s.io/apiextensions-apiserver v0.27.7/go.mod h1:x0p+b5a955lfPz9gaDeBy43obM12s+N9dNHK6+dUL+g= +k8s.io/apimachinery v0.27.8 h1:Xg+ogjDm8s7KmV3vZGf7uOZ0jrC6FPy2Lk/h7BIRmvg= +k8s.io/apimachinery v0.27.8/go.mod h1:EIXLxLt/b1muPITiF5zlrcm7I+YnXsIgM+0GdnPTQvA= +k8s.io/apiserver v0.27.8 h1:fdrEm98wl1lxFoF8tCdQiYQOfmU3TOGkQr4YTLYHwzc= +k8s.io/apiserver v0.27.8/go.mod h1:8yLtDnOdRq9fubaea0zQ3AVheQUinb5Y/RqBUdQMT0Y= +k8s.io/client-go v0.27.8 h1:uXIsGniqc85kKQ8FV0iAwspb0JHtS1UybqrqzHaQ9hc= +k8s.io/client-go v0.27.8/go.mod h1:Ka6MUpObn3LRTfFPvjzyettp8LXCbhqLzZfi8TD4fP8= +k8s.io/component-base v0.27.8 h1:O8YRFv/wWvoo9z62p1N52lq+w5FpzILAlE1h8b9o3K8= +k8s.io/component-base v0.27.8/go.mod h1:h3uyZl+bFQeuLRz3owfSLaw3JKTrn6gmbvybkkW2z+I= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kms v0.26.2 h1:GM1gg3tFK3OUU/QQFi93yGjG3lJT8s8l3Wkn2+VxBLM= -k8s.io/kms v0.26.2/go.mod h1:69qGnf1NsFOQP07fBYqNLZklqEHSJF024JqYCaeVxHg= -k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d h1:VcFq5n7wCJB2FQMCIHfC+f+jNcGgNMar1uKd6rVlifU= -k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= +k8s.io/kms v0.27.8 h1:D7s2vZAFWKD1+1Yg6Y4C+l7k+ZjEsMc7BjY0QRKAOY8= +k8s.io/kms v0.27.8/go.mod h1:+yIM/3y+UgdaZSQaMJObNONp0i237t/dVrxswECh7a0= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= +k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= k8s.io/metrics v0.26.2 h1:2gUvUWWnHPdE2tyA5DvyHC8HGryr+izhY9i5dzLP06s= k8s.io/metrics v0.26.2/go.mod h1:PX1wm9REV9hSGuw9GcXTFNDgab1KRXck3mNeiLYbRho= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= @@ -945,16 +931,15 @@ k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 h1:+xBL5uTc+BkPBwmMi3vYfUJjq+N3K+H6PXeETwf5cPI= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo= -sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= -sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= +sigs.k8s.io/controller-runtime v0.15.3 h1:L+t5heIaI3zeejoIyyvLQs5vTVu/67IU2FfisVzFlBc= +sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds= sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 h1:puQ5YlyBjhxg+OQ1YPMJXwtk7WhC4E6AlWIQ9pC8jws= sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3/go.mod h1:9nUXR/EgdYZto1aQ6yhwOksPR7J979jSyOqic1IgaOo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= From 510a3a82c2cbd3fb4272c797f0aa03354d7aa69e Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 19 Dec 2023 09:33:28 +0000 Subject: [PATCH 263/377] Point the CFO to compatible MCAD --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ec0700b4..a8ab01f2 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 -replace github.com/project-codeflare/multi-cluster-app-dispatcher => github.com/ChristianZaccaria/multi-cluster-app-dispatcher v0.0.0-20231213170206-696368415a5a +replace github.com/project-codeflare/multi-cluster-app-dispatcher => github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e require ( github.com/NYTimes/gziphandler v1.1.1 // indirect diff --git a/go.sum b/go.sum index 81c66b13..0082a662 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,6 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ChristianZaccaria/multi-cluster-app-dispatcher v0.0.0-20231213170206-696368415a5a h1:11bIdGaYS1YOxNYyvR8S9VNNhcMDMvjQxdJ4VMEp810= -github.com/ChristianZaccaria/multi-cluster-app-dispatcher v0.0.0-20231213170206-696368415a5a/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= @@ -393,6 +391,8 @@ github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= github.com/project-codeflare/instascale v0.3.1 h1:LIKo5NaX7kDPmAYy1aAkF0yykql3ZdCiiLsBOC4HRcM= github.com/project-codeflare/instascale v0.3.1/go.mod h1:Fdy3daVhz3BHFvPyr9zfH6uFF2yww73U41Wq2S2zDI8= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e h1:vhauNtZc/4DWVIP+6NyyDifBW9epSxpTbJzV7UKhG9M= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From 944300bd1bfc2bb1ef7420e571e86a51ff7e62ff Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 19 Dec 2023 12:17:03 +0100 Subject: [PATCH 264/377] Remove MCAD replace line in go.mod --- go.mod | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a8ab01f2..0293d7de 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 github.com/project-codeflare/instascale v0.3.1 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 go.uber.org/zap v1.26.0 k8s.io/api v0.27.8 @@ -22,8 +22,6 @@ require ( replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 -replace github.com/project-codeflare/multi-cluster-app-dispatcher => github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e - require ( github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect From 8b6232d44969683821e442b6a9702497196ea85f Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 20 Dec 2023 07:39:07 +0100 Subject: [PATCH 265/377] e2e tests: Export all KinD logs --- .github/workflows/e2e_tests.yaml | 6 ++++++ .github/workflows/olm_tests.yaml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 180bc23c..0786c290 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -92,6 +92,12 @@ jobs: echo "Printing KubeRay operator logs" kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log + - name: Export all KinD pod logs + uses: ./common/github-actions/kind-export-logs + if: always() && steps.deploy.outcome == 'success' + with: + output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} + - name: Upload logs uses: actions/upload-artifact@v3 if: always() && steps.deploy.outcome == 'success' diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index e4716bd0..6889604b 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -185,6 +185,12 @@ jobs: echo "Printing KubeRay operator logs" kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log + - name: Export all KinD pod logs + uses: ./common/github-actions/kind-export-logs + if: always() && steps.deploy.outcome == 'success' + with: + output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} + - name: Upload logs uses: actions/upload-artifact@v3 if: always() && steps.deploy.outcome == 'success' From 80df38e7cb102fed8cb4d03e61eb85260604523d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 21 Dec 2023 16:31:51 +0100 Subject: [PATCH 266/377] Add ChristianZaccaria as reviewer --- OWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS b/OWNERS index efdcada1..a3556780 100644 --- a/OWNERS +++ b/OWNERS @@ -9,6 +9,7 @@ approvers: reviewers: - anishasthana - astefanutti + - ChristianZaccaria - dimakis - jbusche - kpostoffice From 86ca67c95d624a4bd6c1b5ccb4c5db176b2cc2d5 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 21 Dec 2023 16:35:21 +0100 Subject: [PATCH 267/377] Add Fiona-Waters as reviewer --- OWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS b/OWNERS index a3556780..c962aaaf 100644 --- a/OWNERS +++ b/OWNERS @@ -11,6 +11,7 @@ reviewers: - astefanutti - ChristianZaccaria - dimakis + - Fiona-Waters - jbusche - kpostoffice - maxusmusti From 575ac74a98cb0b069e9ffc36df61921cf71c1970 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 22 Dec 2023 10:35:00 +0100 Subject: [PATCH 268/377] Fix MCAD version in Makefile --- .github/workflows/tag-and-build.yml | 4 ++++ Makefile | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 038b77b0..02c850f7 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -109,6 +109,10 @@ jobs: password: ${{ secrets.QUAY_TOKEN }} registry: quay.io + - name: Align go.mod and go.sum dependencies for released components + run: | + make modules + - name: Image Build and Push run: | make build diff --git a/Makefile b/Makefile index 3f0d0f52..53c0d9af 100644 --- a/Makefile +++ b/Makefile @@ -158,7 +158,7 @@ modules: ## Update Go dependencies. go mod tidy .PHONY: build -build: modules fmt vet ## Build manager binary. +build: fmt vet ## Build manager binary. go build \ -ldflags " \ -X 'main.OperatorVersion=$(BUILD_VERSION)' \ @@ -169,7 +169,7 @@ build: modules fmt vet ## Build manager binary. -o bin/manager main.go .PHONY: run -run: modules manifests fmt vet ## Run a controller from your host. +run: manifests fmt vet ## Run a controller from your host. go run ./main.go .PHONY: image-build From c5cc868b0f376cbdf833c532a3d7c94b02c59ee9 Mon Sep 17 00:00:00 2001 From: Kevin Date: Mon, 8 Jan 2024 16:49:31 -0500 Subject: [PATCH 269/377] remove auto add issue workflow Signed-off-by: Kevin --- .github/workflows/auto-add-issues.yml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 .github/workflows/auto-add-issues.yml diff --git a/.github/workflows/auto-add-issues.yml b/.github/workflows/auto-add-issues.yml deleted file mode 100644 index c1e94099..00000000 --- a/.github/workflows/auto-add-issues.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Label new issues as needs-triage and add to CodeFlare Sprint Board -on: - issues: - types: - - opened -jobs: - add-to-project: - name: Add issue to project - runs-on: ubuntu-latest - steps: - - uses: actions/add-to-project@v0.5.0 - with: - project-url: https://github.com/orgs/project-codeflare/projects/8 - github-token: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} From 35fed6730f280ed7797d1d5df33cd7af04965b70 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 10 Jan 2024 09:34:53 +0100 Subject: [PATCH 270/377] Upgrade Kuberay to version v1.0.0 --- Makefile | 2 +- go.mod | 2 +- go.sum | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 53c0d9af..8f15e0b4 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v1.0.0-rc.1 +KUBERAY_VERSION ?= v1.0.0 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 diff --git a/go.mod b/go.mod index 0293d7de..29e863d4 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 github.com/project-codeflare/instascale v0.3.1 github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e - github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 + github.com/ray-project/kuberay/ray-operator v1.0.0 go.uber.org/zap v1.26.0 k8s.io/api v0.27.8 k8s.io/apimachinery v0.27.8 diff --git a/go.sum b/go.sum index 0082a662..4fdd1021 100644 --- a/go.sum +++ b/go.sum @@ -419,8 +419,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1 h1:aoOajXNG2QnTPmnSzbCXLFBXrVk7bzQGrwwVzTXSOzo= -github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1/go.mod h1:NDvscwYbeLSh+Cfc2UTeyPWODtNKPCsPjD/2kg3ZXPw= +github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= +github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= @@ -435,7 +435,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= From 0e7bd88bb15ee2eba29f395e9e7e4c8e1903eaf4 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 11 Jan 2024 12:41:45 +0100 Subject: [PATCH 271/377] Remove SDK e2e test --- test/e2e/install-codeflare-sdk.sh | 22 --- test/e2e/mnist_raycluster_sdk.py | 86 ----------- test/e2e/mnist_raycluster_sdk_test.go | 215 -------------------------- 3 files changed, 323 deletions(-) delete mode 100755 test/e2e/install-codeflare-sdk.sh delete mode 100644 test/e2e/mnist_raycluster_sdk.py delete mode 100644 test/e2e/mnist_raycluster_sdk_test.go diff --git a/test/e2e/install-codeflare-sdk.sh b/test/e2e/install-codeflare-sdk.sh deleted file mode 100755 index e90f4071..00000000 --- a/test/e2e/install-codeflare-sdk.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# go to codeflare-sdk folder and install codeflare-sdk -cd .. - -# Install Poetry and configure virtualenvs -pip install poetry -poetry config virtualenvs.create false - -cd codeflare-sdk -# Clone the CodeFlare SDK repository -git clone --branch main https://github.com/project-codeflare/codeflare-sdk.git - -cd codeflare-sdk - -# Lock dependencies and install them -poetry lock --no-update -poetry install --with test,docs - -# Return to the workdir -cd ../.. -cd workdir diff --git a/test/e2e/mnist_raycluster_sdk.py b/test/e2e/mnist_raycluster_sdk.py deleted file mode 100644 index d171c9b0..00000000 --- a/test/e2e/mnist_raycluster_sdk.py +++ /dev/null @@ -1,86 +0,0 @@ -import sys -import os - -from time import sleep - -from torchx.specs.api import AppState, is_terminal - -from codeflare_sdk.cluster.cluster import Cluster, ClusterConfiguration -from codeflare_sdk.job.jobs import DDPJobDefinition - -namespace = sys.argv[1] -ray_image = os.getenv('RAY_IMAGE') -host = os.getenv('CLUSTER_HOSTNAME') - -ingress_options = {} -if host is not None: - ingress_options = { - "ingresses": [ - { - "ingressName": "ray-dashboard", - "port": 8265, - "pathType": "Prefix", - "path": "/", - "host": host, - }, - ] - } - - -cluster = Cluster(ClusterConfiguration( - name='mnist', - namespace=namespace, - num_workers=1, - head_cpus='500m', - head_memory=2, - min_cpus='500m', - max_cpus=1, - min_memory=0.5, - max_memory=2, - num_gpus=0, - instascale=False, - image=ray_image, - ingress_options=ingress_options, -)) - -cluster.up() - -cluster.status() - -cluster.wait_ready() - -cluster.status() - -cluster.details() - -jobdef = DDPJobDefinition( - name="mnist", - script="mnist.py", - scheduler_args={"requirements": "requirements.txt"}, -) -job = jobdef.submit(cluster) - -done = False -time = 0 -timeout = 900 -while not done: - status = job.status() - if is_terminal(status.state): - break - if not done: - print(status) - if timeout and time >= timeout: - raise TimeoutError(f"job has timed out after waiting {timeout}s") - sleep(5) - time += 5 - -print(f"Job has completed: {status.state}") - -print(job.logs()) - -cluster.down() - -if not status.state == AppState.SUCCEEDED: - exit(1) -else: - exit(0) diff --git a/test/e2e/mnist_raycluster_sdk_test.go b/test/e2e/mnist_raycluster_sdk_test.go deleted file mode 100644 index be6933f6..00000000 --- a/test/e2e/mnist_raycluster_sdk_test.go +++ /dev/null @@ -1,215 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" - - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// Creates a Ray cluster, and trains the MNIST dataset using the CodeFlare SDK. -// Asserts successful completion of the training job. -// -// This covers the installation of the CodeFlare SDK, as well as the RBAC required -// for the SDK to successfully perform requests to the cluster, on behalf of the -// impersonated user. -func TestMNISTRayClusterSDK(t *testing.T) { - test := With(t) - test.T().Parallel() - - // Create a namespace - namespace := test.NewTestNamespace() - - // Test configuration - config := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // SDK script - "mnist_raycluster_sdk.py": ReadFile(test, "mnist_raycluster_sdk.py"), - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - // codeflare-sdk installation script - "install-codeflare-sdk.sh": ReadFile(test, "install-codeflare-sdk.sh"), - }) - - // Create RBAC, retrieve token for user with limited rights - policyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1.GroupVersion.Group}, - Resources: []string{"rayclusters", "rayclusters/status"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"route.openshift.io"}, - Resources: []string{"routes"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"ingresses"}, - }, - } - - // Create cluster wide RBAC, required for SDK OpenShift check - // TODO reevaluate once SDK change OpenShift detection logic - clusterPolicyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"config.openshift.io"}, - Resources: []string{"ingresses"}, - ResourceNames: []string{"cluster"}, - }, - } - - sa := CreateServiceAccount(test, namespace.Name) - role := CreateRole(test, namespace.Name, policyRules) - CreateRoleBinding(test, namespace.Name, sa, role) - clusterRole := CreateClusterRole(test, clusterPolicyRules) - CreateClusterRoleBinding(test, sa, clusterRole) - - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "sdk", - Namespace: namespace.Name, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - BackoffLimit: Ptr(int32(0)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - // FIXME: switch to base Python image once the dependency on OpenShift CLI is removed - // See https://github.com/project-codeflare/codeflare-sdk/pull/146 - Image: "quay.io/opendatahub/notebooks:jupyter-minimal-ubi8-python-3.8-4c8f26e", - Env: []corev1.EnvVar{ - {Name: "PYTHONUSERBASE", Value: "/workdir"}, - {Name: "RAY_IMAGE", Value: GetRayImage()}, - }, - Command: []string{"/bin/sh", "-c", "cp /test/* . && chmod +x install-codeflare-sdk.sh && ./install-codeflare-sdk.sh && python mnist_raycluster_sdk.py" + " " + namespace.Name}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - { - Name: "codeflare-sdk", - MountPath: "/codeflare-sdk", - }, - { - Name: "workdir", - MountPath: "/workdir", - }, - }, - WorkingDir: "/workdir", - SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: Ptr(false), - SeccompProfile: &corev1.SeccompProfile{ - Type: "RuntimeDefault", - }, - Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - RunAsNonRoot: Ptr(true), - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - { - Name: "codeflare-sdk", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "workdir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - ServiceAccountName: sa.Name, - }, - }, - }, - } - if GetClusterType(test) == KindCluster { - // Take first KinD node and redirect pod hostname requests there - node := GetNodes(test)[0] - hostname := GetClusterHostname(test) - IP := GetNodeInternalIP(test, node) - - test.T().Logf("Setting KinD cluster hostname '%s' to node IP '%s' for SDK pod", hostname, IP) - job.Spec.Template.Spec.HostAliases = []corev1.HostAlias{ - { - IP: IP, - Hostnames: []string{hostname}, - }, - } - - // Propagate hostname into Python code as env variable - hostnameEnvVar := corev1.EnvVar{Name: "CLUSTER_HOSTNAME", Value: hostname} - job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, hostnameEnvVar) - } - - job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Create(test.Ctx(), job, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created Job %s/%s successfully", job.Namespace, job.Name) - - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( - Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), - )) - - // Assert the job has completed successfully - test.Expect(GetJob(test, job.Namespace, job.Name)). - To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue))) -} From 233de7ea5f895ef5be319386f437f46d3bdde048 Mon Sep 17 00:00:00 2001 From: Mark Campbell Date: Fri, 19 Jan 2024 12:34:36 +0000 Subject: [PATCH 272/377] Added Workflow job to update CFO image (#398) --- .github/workflows/tag-and-build.yml | 7 ++++++- Makefile | 4 ++-- config/manager/kustomization.yaml | 16 ++++++++++++++++ config/manager/manager.yaml | 2 +- config/manager/params.env | 1 + config/manager/params.yaml | 3 +++ config/manifests/kustomization.yaml | 3 ++- 7 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 config/manager/params.env create mode 100644 config/manager/params.yaml diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 02c850f7..f04d43d6 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -102,6 +102,11 @@ jobs: sed -i -E "s/(.*INSTASCALE_VERSION \?= ).*/\1${{ github.event.inputs.instascale-version }}/" Makefile sed -i -E "s/(.*KUBERAY_VERSION \?= ).*/\1${{ github.event.inputs.kuberay-version }}/" Makefile + - name: Update image version in params.env + run: | + VERSION=${{ github.event.inputs.version }} perl -i -pe 's/:(.*)$/:$ENV{"VERSION"}/' config/manager/params.env + shell: bash + - name: Login to Quay.io uses: redhat-actions/podman-login@v1 with: @@ -138,7 +143,7 @@ jobs: uses: stefanzweifel/git-auto-commit-action@v4 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} - file_pattern: 'README.md *.yaml Makefile go.mod go.sum' + file_pattern: 'README.md *.yaml Makefile go.mod go.sum *.env' create_branch: true branch: ${{ env.PR_BRANCH_NAME }} diff --git a/Makefile b/Makefile index 8f15e0b4..0e2b24c3 100644 --- a/Makefile +++ b/Makefile @@ -198,7 +198,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + cd config/manager && IMAGE=$(IMG) perl -i -pe 's/codeflare-operator-controller-image=(.*)$$/codeflare-operator-controller-image=$$ENV{"IMAGE"}/' params.env $(KUSTOMIZE) build config/${ENV} | kubectl apply -f - git restore config/* @@ -283,8 +283,8 @@ validate-bundle: install-operator-sdk .PHONY: bundle bundle: manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. + cd config/manager && IMAGE=$(IMG) perl -i -pe 's/codeflare-operator-controller-image=(.*)$$/codeflare-operator-controller-image=$$ENV{"IMAGE"}/' params.env $(OPERATOR_SDK) generate kustomize manifests -q - cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 4fe5ccaa..847eca17 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -3,3 +3,19 @@ resources: generatorOptions: disableNameSuffixHash: true + +configMapGenerator: +- name: stack-config + envs: + - params.env +configurations: + - params.yaml + +vars: +- name: codeflare_operator_controller_image + objref: + kind: ConfigMap + name: stack-config + apiVersion: v1 + fieldref: + fieldpath: data.codeflare-operator-controller-image diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 8a4f6451..20a34fe9 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -34,7 +34,7 @@ spec: containers: - command: - /manager - image: controller:latest + image: $(codeflare_operator_controller_image) imagePullPolicy: Always name: manager securityContext: diff --git a/config/manager/params.env b/config/manager/params.env new file mode 100644 index 00000000..8c9de784 --- /dev/null +++ b/config/manager/params.env @@ -0,0 +1 @@ +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.0.1 diff --git a/config/manager/params.yaml b/config/manager/params.yaml new file mode 100644 index 00000000..43509ff2 --- /dev/null +++ b/config/manager/params.yaml @@ -0,0 +1,3 @@ +varReference: + - path: spec/template/spec/containers[]/image + kind: Deployment diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml index 958a9e24..161a419d 100644 --- a/config/manifests/kustomization.yaml +++ b/config/manifests/kustomization.yaml @@ -1,8 +1,9 @@ # These resources constitute the fully configured set of manifests # used to generate the 'manifests/' directory in a bundle. resources: -- bases/codeflare-operator.clusterserviceversion.yaml - ../default - ../scorecard +- bases/codeflare-operator.clusterserviceversion.yaml + apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization From 4416347aeed370987e933cd04bc58de555e66a84 Mon Sep 17 00:00:00 2001 From: Dimitri Saridakis Date: Fri, 19 Jan 2024 13:44:22 +0000 Subject: [PATCH 273/377] Update project-codeflare-release.yml (#445) --- .github/workflows/project-codeflare-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 77d0a0a1..59f24be0 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -70,7 +70,7 @@ jobs: - name: Release MCAD run: | - gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref ${{ github.ref }} --field tag=${{ github.event.inputs.mcad-version }} + gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref go-1.20 --field tag=${{ github.event.inputs.mcad-version }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash From 003d61b3ae4ffe5c31cea8071d8d3614c0bdbcc5 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account Date: Fri, 19 Jan 2024 14:26:58 +0000 Subject: [PATCH 274/377] Update dependency versions for release v1.1.0 --- Makefile | 4 ++-- README.md | 10 +++++----- config/crd/mcad/kustomization.yaml | 2 +- config/manager/params.env | 2 +- go.mod | 5 ++--- go.sum | 11 ++++------- 6 files changed, 15 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 0e2b24c3..dccaea28 100644 --- a/Makefile +++ b/Makefile @@ -12,11 +12,11 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.3.1 +INSTASCALE_VERSION ?= v0.4.0 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.38.1 +MCAD_VERSION ?= v1.39.0 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} diff --git a/README.md b/README.md index c3048b29..9488b3b4 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.0.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.0.1) | -| Multi-Cluster App Dispatcher | [v1.38.1](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.38.1) | -| CodeFlare-SDK | [v0.12.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.12.1) | -| InstaScale | [v0.3.1](https://github.com/project-codeflare/instascale/releases/tag/v0.3.1) | -| KubeRay | [v1.0.0-rc.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0-rc.0) | +| CodeFlare Operator | [v1.1.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.1.0) | +| Multi-Cluster App Dispatcher | [v1.39.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.39.0) | +| CodeFlare-SDK | [v0.13.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.13.0) | +| InstaScale | [v0.4.0](https://github.com/project-codeflare/instascale/releases/tag/v0.4.0) | +| KubeRay | [v1.0.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0) | ## Development diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index 3cd50c48..0e4ebfa7 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.38.1 +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.39.0 diff --git a/config/manager/params.env b/config/manager/params.env index 8c9de784..ace2f2f2 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1 +1 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.0.1 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.1.0 diff --git a/go.mod b/go.mod index 29e863d4..149ba772 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 - github.com/project-codeflare/instascale v0.3.1 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e + github.com/project-codeflare/instascale v0.4.0 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 github.com/ray-project/kuberay/ray-operator v1.0.0 go.uber.org/zap v1.26.0 k8s.io/api v0.27.8 @@ -116,7 +116,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.27.7 // indirect k8s.io/apiserver v0.27.8 // indirect - k8s.io/klog v1.0.0 // indirect k8s.io/kms v0.27.8 // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect k8s.io/metrics v0.26.2 // indirect diff --git a/go.sum b/go.sum index 4fdd1021..797c519b 100644 --- a/go.sum +++ b/go.sum @@ -125,7 +125,6 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= @@ -389,10 +388,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 h1:81+ma1mchF/LtAGsf+poAt50kJ/fLYjoTAcZOxci1Yc= github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= -github.com/project-codeflare/instascale v0.3.1 h1:LIKo5NaX7kDPmAYy1aAkF0yykql3ZdCiiLsBOC4HRcM= -github.com/project-codeflare/instascale v0.3.1/go.mod h1:Fdy3daVhz3BHFvPyr9zfH6uFF2yww73U41Wq2S2zDI8= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e h1:vhauNtZc/4DWVIP+6NyyDifBW9epSxpTbJzV7UKhG9M= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.2-0.20231218174414-5b7ee85b759e/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= +github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo= +github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 h1:zoS7pEAWK6eGELPCIIHB3W8Zb/a27Rf55ChYso7EV3o= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -916,8 +915,6 @@ k8s.io/client-go v0.27.8 h1:uXIsGniqc85kKQ8FV0iAwspb0JHtS1UybqrqzHaQ9hc= k8s.io/client-go v0.27.8/go.mod h1:Ka6MUpObn3LRTfFPvjzyettp8LXCbhqLzZfi8TD4fP8= k8s.io/component-base v0.27.8 h1:O8YRFv/wWvoo9z62p1N52lq+w5FpzILAlE1h8b9o3K8= k8s.io/component-base v0.27.8/go.mod h1:h3uyZl+bFQeuLRz3owfSLaw3JKTrn6gmbvybkkW2z+I= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kms v0.27.8 h1:D7s2vZAFWKD1+1Yg6Y4C+l7k+ZjEsMc7BjY0QRKAOY8= From b584b5eeb127b7367dbcc3fdbc119704729bbef6 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Fri, 19 Jan 2024 16:35:01 +0000 Subject: [PATCH 275/377] update: change CSV alm-examples to '' --- .../bases/codeflare-operator.clusterserviceversion.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml index 77295679..c3db15ce 100644 --- a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml @@ -2,7 +2,7 @@ apiVersion: operators.coreos.com/v1alpha1 kind: ClusterServiceVersion metadata: annotations: - alm-examples: '[]' + alm-examples: '' capabilities: Basic Install categories: AI/Machine Learning, Big Data operatorframework.io/suggested-namespace: openshift-operators From a040aa637e0b47007ec33ef52028619fd627cb34 Mon Sep 17 00:00:00 2001 From: Christian Zaccaria <73656840+ChristianZaccaria@users.noreply.github.com> Date: Fri, 19 Jan 2024 17:42:35 +0000 Subject: [PATCH 276/377] Revert "update: change CSV alm-examples to ''" (#448) --- .../bases/codeflare-operator.clusterserviceversion.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml index c3db15ce..77295679 100644 --- a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml @@ -2,7 +2,7 @@ apiVersion: operators.coreos.com/v1alpha1 kind: ClusterServiceVersion metadata: annotations: - alm-examples: '' + alm-examples: '[]' capabilities: Basic Install categories: AI/Machine Learning, Big Data operatorframework.io/suggested-namespace: openshift-operators From a4cf5752b1a3ba32a8b14786b876a724b0dd4605 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Mon, 22 Jan 2024 09:17:53 +0000 Subject: [PATCH 277/377] Fixed MCAD release --- .github/workflows/project-codeflare-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 59f24be0..77d0a0a1 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -70,7 +70,7 @@ jobs: - name: Release MCAD run: | - gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref go-1.20 --field tag=${{ github.event.inputs.mcad-version }} + gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref ${{ github.ref }} --field tag=${{ github.event.inputs.mcad-version }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash From 89efe670eb274e7a5f6cb91060a3bf90f6df71a5 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Mon, 22 Jan 2024 11:18:27 +0000 Subject: [PATCH 278/377] update: change CSV alm-examples Signed-off-by: Eoin Gallinagh --- ...eflare-operator.clusterserviceversion.yaml | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml index 77295679..5531e61a 100644 --- a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml @@ -2,7 +2,45 @@ apiVersion: operators.coreos.com/v1alpha1 kind: ClusterServiceVersion metadata: annotations: - alm-examples: '[]' + alm-examples: |- + [{ + "apiVersion": "workload.codeflare.dev/v1beta1", + "kind": "AppWrapper", + "metadata": {"name": "0001-aw-generic-deployment-1"}, + "spec": { + "resources": { + "GenericItems": [{ + "replicas": 1, + "generictemplate": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "0001-aw-generic-deployment-1", + "labels": {"app": "0001-aw-generic-deployment-1"} + }, + "spec": { + "selector": {"matchLabels": {"app": "0001-aw-generic-deployment-1"}}, + "replicas": 2, + "template": { + "metadata": {"labels": {"app": "0001-aw-generic-deployment-1"}}, + "spec": { + "containers": [{ + "name": "0001-aw-generic-deployment-1", + "image": "kicbase/echo-server:1.0", + "ports": [{"containerPort": 80}], + "resources": { + "requests": {"cpu": "100m", "memory": "256Mi"}, + "limits": {"cpu": "100m", "memory": "256Mi"} + } + }] + } + } + } + } + }] + } + } + }] capabilities: Basic Install categories: AI/Machine Learning, Big Data operatorframework.io/suggested-namespace: openshift-operators From 0cf73e844491fa7f995af611b4b0f2680e0a4ef3 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 10 Jan 2024 13:13:47 +0100 Subject: [PATCH 279/377] Download MNIST dataset from specific location --- go.mod | 2 +- go.sum | 4 +- test/e2e/mnist.py | 33 ++++++++++++++- test/e2e/mnist_pytorch_mcad_job_test.go | 3 +- test/e2e/mnist_rayjob_mcad_raycluster_test.go | 42 ++++++++++--------- test/upgrade/olm_upgrade_test.go | 3 +- 6 files changed, 61 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 149ba772..fbac9a7f 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 - github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 + github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717 github.com/project-codeflare/instascale v0.4.0 github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 github.com/ray-project/kuberay/ray-operator v1.0.0 diff --git a/go.sum b/go.sum index 797c519b..e501c235 100644 --- a/go.sum +++ b/go.sum @@ -386,8 +386,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069 h1:81+ma1mchF/LtAGsf+poAt50kJ/fLYjoTAcZOxci1Yc= -github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069/go.mod h1:zdi2GCYJX+QyxFWyCLMoTme3NMz/aucWDJWMqKfigxk= +github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717 h1:knUKEKvfEzVuSwQ4NAe2+I/Oxo4WztU5rYR8d/F66Lw= +github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo= github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo= github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 h1:zoS7pEAWK6eGELPCIIHB3W8Zb/a27Rf55ChYso7EV3o= diff --git a/test/e2e/mnist.py b/test/e2e/mnist.py index 244c84d2..4cfe0b43 100644 --- a/test/e2e/mnist.py +++ b/test/e2e/mnist.py @@ -15,6 +15,7 @@ import os import torch +import requests from pytorch_lightning import LightningModule, Trainer from pytorch_lightning.callbacks.progress import TQDMProgressBar from torch import nn @@ -32,6 +33,8 @@ print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) +print("MNIST_DATASET_URL: is ", os.getenv("MNIST_DATASET_URL")) +MNIST_DATASET_URL = os.getenv("MNIST_DATASET_URL") class LitMNIST(LightningModule): def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4): @@ -110,8 +113,34 @@ def configure_optimizers(self): #################### def prepare_data(self): - # download - print("Downloading MNIST dataset...") + datasetFiles = [ + "t10k-images-idx3-ubyte", + "t10k-labels-idx1-ubyte", + "train-images-idx3-ubyte", + "train-labels-idx1-ubyte" + ] + + # Create required folder structure + downloadLocation = os.path.join(PATH_DATASETS, "MNIST", "raw") + os.makedirs(downloadLocation, exist_ok=True) + print(f"{downloadLocation} folder_path created!") + + for file in datasetFiles: + print(f"Downloading MNIST dataset {file}... to path : {downloadLocation}") + response = requests.get(f"{MNIST_DATASET_URL}{file}", stream=True) + filePath = os.path.join(downloadLocation, file) + + #to download dataset file + try: + if response.status_code == 200: + open(filePath, 'wb').write(response.content) + print(f"{file}: Downloaded and saved zipped file to path - {filePath}") + else: + print(f"Failed to download file {file}") + except Exception as e: + print(e) + print(f"Downloaded MNIST dataset to... {downloadLocation}") + MNIST(self.data_dir, train=True, download=True) MNIST(self.data_dir, train=False, download=True) diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index cbe55fed..883457e9 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -79,7 +79,8 @@ func TestMNISTPyTorchMCAD(t *testing.T) { Name: "job", Image: GetPyTorchImage(), Env: []corev1.EnvVar{ - corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index b8d3f4d0..21bd98ad 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -17,7 +17,6 @@ limitations under the License. package e2e import ( - "encoding/base64" "testing" . "github.com/onsi/gomega" @@ -143,13 +142,6 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { RayStartParams: map[string]string{}, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - InitContainers: []corev1.Container{ - { - Name: "init-myservice", - Image: "busybox:1.28", - Command: []string{"sh", "-c", "until nslookup $RAY_IP.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"}, - }, - }, Containers: []corev1.Container{ { Name: "ray-worker", @@ -230,21 +222,29 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, Spec: rayv1.RayJobSpec{ Entrypoint: "python /home/ray/jobs/mnist.py", - RuntimeEnv: base64.StdEncoding.EncodeToString([]byte(` -{ - "pip": [ - "pytorch_lightning==1.5.10", - "torchmetrics==0.9.1", - "torchvision==0.12.0" - ], - "env_vars": { - } -} -`)), + RuntimeEnvYAML: ` + pip: + - pytorch_lightning==1.5.10 + - torchmetrics==0.9.1 + - torchvision==0.12.0 + env_vars: + MNIST_DATASET_URL: "` + GetMnistDatasetURL() + `" +`, ClusterSelector: map[string]string{ RayJobDefaultClusterSelectorKey: rayCluster.Name, }, ShutdownAfterJobFinishes: false, + SubmitterPodTemplate: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Image: GetRayImage(), + Name: "rayjob-submitter-pod", + }, + }, + }, + }, }, } rayJob, err = test.Client().Ray().RayV1().RayJobs(namespace.Name).Create(test.Ctx(), rayJob, metav1.CreateOptions{}) @@ -256,6 +256,10 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) rayClient := NewRayClusterClient(rayDashboardURL) + // Wait for Ray job id to be available, this value is needed for writing logs in defer + test.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutShort). + Should(WithTransform(RayJobId, Not(BeEmpty()))) + // Retrieving the job logs once it has completed or timed out defer WriteRayJobAPILogs(test, rayClient, GetRayJobId(test, rayJob.Namespace, rayJob.Name)) diff --git a/test/upgrade/olm_upgrade_test.go b/test/upgrade/olm_upgrade_test.go index ac0783e0..8fb4e65f 100644 --- a/test/upgrade/olm_upgrade_test.go +++ b/test/upgrade/olm_upgrade_test.go @@ -96,7 +96,8 @@ func TestMNISTCreateAppWrapper(t *testing.T) { Name: "job", Image: GetPyTorchImage(), Env: []corev1.EnvVar{ - corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ From a40442c3544d82d93b366ca912c971f5fd550bc8 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 25 Jan 2024 11:50:16 +0000 Subject: [PATCH 280/377] Upgrade protobuf dependency to address CVE --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fbac9a7f..41b42bf3 100644 --- a/go.mod +++ b/go.mod @@ -109,7 +109,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/grpc v1.58.3 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index e501c235..e9ce7115 100644 --- a/go.sum +++ b/go.sum @@ -865,8 +865,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 195c9ecae7928f4c29fe8ce3b6c16fae2ae45b1f Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 25 Jan 2024 13:56:38 -0500 Subject: [PATCH 281/377] add roles for admin and editor to operator Signed-off-by: Kevin --- config/rbac/admin_role.yaml | 22 ++++++++++++++++++++++ config/rbac/editor_role.yaml | 24 ++++++++++++++++++++++++ config/rbac/kustomization.yaml | 2 ++ 3 files changed, 48 insertions(+) create mode 100644 config/rbac/admin_role.yaml create mode 100644 config/rbac/editor_role.yaml diff --git a/config/rbac/admin_role.yaml b/config/rbac/admin_role.yaml new file mode 100644 index 00000000..b4f4728c --- /dev/null +++ b/config/rbac/admin_role.yaml @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: clusterrole-admin + labels: + rbac.authorization.kubernetes.io/aggregate-to-admin: "true" +rules: +- apiGroups: + - quota.codeflare.dev + resources: + - quotasubtrees + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch diff --git a/config/rbac/editor_role.yaml b/config/rbac/editor_role.yaml new file mode 100644 index 00000000..aa341297 --- /dev/null +++ b/config/rbac/editor_role.yaml @@ -0,0 +1,24 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: clusterrole-edit + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" +rules: +- apiGroups: + - workload.codeflare.dev + resources: + - schedulingspecs + - appwrappers + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index cdb307c8..a74a16f1 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -4,6 +4,8 @@ resources: # if your manager will use a service account that exists at # runtime. Be sure to update RoleBinding and ClusterRoleBinding # subjects if changing service account names. +- admin_role.yaml +- editor_role.yaml - service_account.yaml - role.yaml - role_binding.yaml From f52d75d43628cfaffd7c6698f8abcb26e1554794 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 26 Jan 2024 16:48:15 +0000 Subject: [PATCH 282/377] Update dependency to address CVE --- go.mod | 21 ++++++++++----------- go.sum | 42 ++++++++++++++++++++---------------------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 41b42bf3..c8cca86f 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,6 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/microcosm-cc/bluemonday v1.0.18 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -75,10 +74,10 @@ require ( github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.15.1 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.46.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect @@ -95,13 +94,13 @@ require ( go.opentelemetry.io/otel/trace v1.19.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/crypto v0.18.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index e9ce7115..a1ad6ba0 100644 --- a/go.sum +++ b/go.sum @@ -341,8 +341,6 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= @@ -397,27 +395,27 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= +github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -537,8 +535,8 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -613,16 +611,16 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -692,13 +690,13 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -708,8 +706,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From ffbd647ab39d3075515c191d2f217d217eb0ce19 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 30 Jan 2024 17:17:37 +0000 Subject: [PATCH 283/377] Add Host to Container port mapping for KinD Cluster --- test/e2e/kind.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh index 5b39a164..db53893c 100755 --- a/test/e2e/kind.sh +++ b/test/e2e/kind.sh @@ -24,6 +24,10 @@ apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane image: kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1 + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP kubeadmConfigPatches: - | kind: InitConfiguration From f8839e9269aa16b0875f5ee42ec6d622bea71888 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 31 Jan 2024 15:41:02 +0100 Subject: [PATCH 284/377] Raise KinD Ingress controller version to v1.9.6 for manual setup --- test/e2e/kind.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/kind.sh b/test/e2e/kind.sh index db53893c..7b7e9053 100755 --- a/test/e2e/kind.sh +++ b/test/e2e/kind.sh @@ -15,7 +15,7 @@ # limitations under the License. set -euo pipefail -: "${INGRESS_NGINX_VERSION:=controller-v1.6.4}" +: "${INGRESS_NGINX_VERSION:=controller-v1.9.6}" echo "Creating KinD cluster" cat < Date: Wed, 24 Jan 2024 23:02:10 +0000 Subject: [PATCH 285/377] Added e2e test for nodepools --- test/e2e/instascale_app_wrapper.go | 1 + test/e2e/instascale_nodepool_test.go | 66 ++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 test/e2e/instascale_nodepool_test.go diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go index e030817f..45305dda 100644 --- a/test/e2e/instascale_app_wrapper.go +++ b/test/e2e/instascale_app_wrapper.go @@ -48,6 +48,7 @@ func instaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *cor Image: GetPyTorchImage(), Env: []corev1.EnvVar{ {Name: "PYTHONUSERBASE", Value: "/workdir"}, + {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, Args: []string{"$PYTHONUSERBASE"}, diff --git a/test/e2e/instascale_nodepool_test.go b/test/e2e/instascale_nodepool_test.go new file mode 100644 index 00000000..17945040 --- /dev/null +++ b/test/e2e/instascale_nodepool_test.go @@ -0,0 +1,66 @@ +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + . "github.com/project-codeflare/codeflare-common/support" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestInstascaleNodepool(t *testing.T) { + + test := With(t) + test.T().Parallel() + + clusterType := GetClusterType(test) + if clusterType != HypershiftCluster { + test.T().Skipf("Skipping test as not running on an Hypershift cluster, resolved cluster type: %s", clusterType) + } + + namespace := test.NewTestNamespace() + + // Test configuration + cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }) + + //create OCM connection + connection := CreateOCMConnection(test) + defer connection.Close() + + // check existing cluster resources + // look for a node pool with a label key equal to aw name - expect NOT to find it + test.Expect(GetNodePools(test, connection)). + ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + + // Setup batch job and AppWrapper + aw := instaScaleJobAppWrapper(test, namespace, cm) + + // apply AppWrapper to cluster + _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) + + // assert that AppWrapper goes to "Running" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + // look for a node pool with a label key equal to aw name - expect to find it + test.Eventually(NodePools(test, connection), TestTimeoutLong). + Should(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + + // assert that the AppWrapper goes to "Completed" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) + + // look for a node pool with a label key equal to aw name - expect NOT to find it + test.Eventually(NodePools(test, connection), TestTimeoutLong). + ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + +} From d85339bfdf1041fd14524797a3a0312dae81280e Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Wed, 31 Jan 2024 13:38:35 +0000 Subject: [PATCH 286/377] address feedback --- test/e2e/instascale_nodepool_test.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/test/e2e/instascale_nodepool_test.go b/test/e2e/instascale_nodepool_test.go index 17945040..8721663d 100644 --- a/test/e2e/instascale_nodepool_test.go +++ b/test/e2e/instascale_nodepool_test.go @@ -1,6 +1,7 @@ package e2e import ( + "fmt" "testing" . "github.com/onsi/gomega" @@ -34,14 +35,15 @@ func TestInstascaleNodepool(t *testing.T) { connection := CreateOCMConnection(test) defer connection.Close() - // check existing cluster resources - // look for a node pool with a label key equal to aw name - expect NOT to find it - test.Expect(GetNodePools(test, connection)). - ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) - // Setup batch job and AppWrapper aw := instaScaleJobAppWrapper(test, namespace, cm) + expectedLabel := fmt.Sprintf("%s-%s", aw.Name, aw.Namespace) + // check existing cluster resources + // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect NOT to find it + test.Expect(GetNodePools(test, connection)). + ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) + // apply AppWrapper to cluster _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) @@ -51,16 +53,16 @@ func TestInstascaleNodepool(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - // look for a node pool with a label key equal to aw name - expect to find it + // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect to find it test.Eventually(NodePools(test, connection), TestTimeoutLong). - Should(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + Should(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) // assert that the AppWrapper goes to "Completed" state test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - // look for a node pool with a label key equal to aw name - expect NOT to find it + // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect NOT to find it test.Eventually(NodePools(test, connection), TestTimeoutLong). - ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(HavePrefix("test-instascale"))))) + ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) } From 1264faabc1835b3aafb5e11fc31c4b1bbb9a1382 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 1 Feb 2024 15:33:52 +0100 Subject: [PATCH 287/377] Adjust existing e2e tests to provide custom Pypi index URL --- README.md | 15 +++++++ go.mod | 2 +- go.sum | 4 +- test/e2e/mnist_pytorch_mcad_job_test.go | 2 + test/e2e/mnist_rayjob_mcad_raycluster_test.go | 2 + test/odh/mcad_ray_test.go | 42 ++++++++++++++++++- test/odh/notebook.go | 12 ++---- test/odh/resources/custom-nb-small.yaml | 2 +- test/odh/resources/mnist.py | 33 ++++++++++++++- test/odh/resources/mnist_ray_mini.ipynb | 5 ++- test/odh/resources/requirements.txt | 2 + test/odh/template.go | 40 ++++++++++++++++++ 12 files changed, 143 insertions(+), 18 deletions(-) create mode 100644 test/odh/template.go diff --git a/README.md b/README.md index 9488b3b4..de350d04 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,21 @@ The e2e tests can be executed locally by running the following commands: Alternatively, You can run the e2e test(s) from your IDE / debugger. +#### Testing on disconnected cluster + +To properly run e2e tests on disconnected cluster user has to provide additional environment variables to properly configure testing environment: + +- `CODEFLARE_TEST_PYTORCH_IMAGE` - image tag for image used to run training job using MCAD +- `CODEFLARE_TEST_RAY_IMAGE` - image tag for Ray cluster image +- `MNIST_DATASET_URL` - URL where MNIST dataset is available +- `PIP_INDEX_URL` - URL where PyPI server with needed dependencies is running +- `PIP_TRUSTED_HOST` - PyPI server hostname + +For ODH tests additional environment variables are needed: + +- `NOTEBOOK_IMAGE_STREAM_NAME` - name of the ODH Notebook ImageStream to be used +- `ODH_NAMESPACE` - namespace where ODH is installed + ## Release 1. Invoke [project-codeflare-release.yaml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/project-codeflare-release.yml) diff --git a/go.mod b/go.mod index c8cca86f..252653db 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 - github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717 + github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 github.com/project-codeflare/instascale v0.4.0 github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 github.com/ray-project/kuberay/ray-operator v1.0.0 diff --git a/go.sum b/go.sum index a1ad6ba0..b36099d6 100644 --- a/go.sum +++ b/go.sum @@ -384,8 +384,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717 h1:knUKEKvfEzVuSwQ4NAe2+I/Oxo4WztU5rYR8d/F66Lw= -github.com/project-codeflare/codeflare-common v0.0.0-20240111082724-8f0684651717/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= +github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 h1:30LG8751WElZmWA3mVS8l23l2oZnUCqbDkLCyy0U/p0= +github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo= github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo= github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 h1:zoS7pEAWK6eGELPCIIHB3W8Zb/a27Rf55ChYso7EV3o= diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go index 883457e9..a642d31c 100644 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ b/test/e2e/mnist_pytorch_mcad_job_test.go @@ -81,6 +81,8 @@ func TestMNISTPyTorchMCAD(t *testing.T) { Env: []corev1.EnvVar{ {Name: "PYTHONUSERBASE", Value: "/workdir"}, {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, + {Name: "PIP_INDEX_URL", Value: GetPipIndexURL()}, + {Name: "PIP_TRUSTED_HOST", Value: GetPipTrustedHost()}, }, Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, VolumeMounts: []corev1.VolumeMount{ diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_mcad_raycluster_test.go index 21bd98ad..1118079e 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_mcad_raycluster_test.go @@ -229,6 +229,8 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { - torchvision==0.12.0 env_vars: MNIST_DATASET_URL: "` + GetMnistDatasetURL() + `" + PIP_INDEX_URL: "` + GetPipIndexURL() + `" + PIP_TRUSTED_HOST: "` + GetPipTrustedHost() + `" `, ClusterSelector: map[string]string{ RayJobDefaultClusterSelectorKey: rayCluster.Name, diff --git a/test/odh/mcad_ray_test.go b/test/odh/mcad_ray_test.go index 770b64d9..b72456c5 100644 --- a/test/odh/mcad_ray_test.go +++ b/test/odh/mcad_ray_test.go @@ -38,8 +38,8 @@ func TestMCADRay(t *testing.T) { config := CreateConfigMap(test, namespace.Name, map[string][]byte{ // MNIST Ray Notebook jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_ray_mini.ipynb"), - "mnist.py": ReadFile(test, "resources/mnist.py"), - "requirements.txt": ReadFile(test, "resources/requirements.txt"), + "mnist.py": readMnistPy(test), + "requirements.txt": readRequirementsTxt(test), }) // Create RBAC, retrieve token for user with limited rights @@ -59,6 +59,11 @@ func TestMCADRay(t *testing.T) { APIGroups: []string{"route.openshift.io"}, Resources: []string{"routes"}, }, + { + Verbs: []string{"get", "list"}, + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"ingresses"}, + }, } // Create cluster wide RBAC, required for SDK OpenShift check @@ -96,3 +101,36 @@ func TestMCADRay(t *testing.T) { test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). Should(HaveLen(0)) } + +func readRequirementsTxt(test Test) []byte { + // Read the requirements.txt from resources and perform replacements for custom values using go template + props := struct { + PipIndexUrl string + PipTrustedHost string + }{ + PipIndexUrl: "--index " + GetPipIndexURL(), + } + + // Provide trusted host only if defined + if len(GetPipTrustedHost()) > 0 { + props.PipTrustedHost = "--trusted-host " + GetPipTrustedHost() + } + + template, err := files.ReadFile("resources/requirements.txt") + test.Expect(err).NotTo(HaveOccurred()) + + return ParseTemplate(test, template, props) +} + +func readMnistPy(test Test) []byte { + // Read the mnist.py from resources and perform replacements for custom values using go template + props := struct { + MnistDatasetURL string + }{ + MnistDatasetURL: GetMnistDatasetURL(), + } + template, err := files.ReadFile("resources/mnist.py") + test.Expect(err).NotTo(HaveOccurred()) + + return ParseTemplate(test, template, props) +} diff --git a/test/odh/notebook.go b/test/odh/notebook.go index 8c7b2827..70bdda82 100644 --- a/test/odh/notebook.go +++ b/test/odh/notebook.go @@ -18,7 +18,6 @@ package odh import ( "bytes" - "html/template" gomega "github.com/onsi/gomega" . "github.com/project-codeflare/codeflare-common/support" @@ -44,6 +43,7 @@ type NotebookProps struct { OpenDataHubNamespace string ImageStreamName string ImageStreamTag string + RayImage string NotebookConfigMapName string NotebookConfigMapFileName string NotebookPVC string @@ -66,23 +66,19 @@ func createNotebook(test Test, namespace *corev1.Namespace, notebookToken, jupyt OpenDataHubNamespace: GetOpenDataHubNamespace(), ImageStreamName: GetNotebookImageStreamName(test), ImageStreamTag: recommendedTagName, + RayImage: GetRayImage(), NotebookConfigMapName: jupyterNotebookConfigMapName, NotebookConfigMapFileName: jupyterNotebookConfigMapFileName, NotebookPVC: notebookPVC.Name, } notebookTemplate, err := files.ReadFile("resources/custom-nb-small.yaml") test.Expect(err).NotTo(gomega.HaveOccurred()) - parsedNotebookTemplate, err := template.New("notebook").Parse(string(notebookTemplate)) - test.Expect(err).NotTo(gomega.HaveOccurred()) - // Filter template and store results to the buffer - notebookBuffer := new(bytes.Buffer) - err = parsedNotebookTemplate.Execute(notebookBuffer, notebookProps) - test.Expect(err).NotTo(gomega.HaveOccurred()) + parsedNotebookTemplate := ParseTemplate(test, notebookTemplate, notebookProps) // Create Notebook CR notebookCR := &unstructured.Unstructured{} - err = yaml.NewYAMLOrJSONDecoder(notebookBuffer, 8192).Decode(notebookCR) + err = yaml.NewYAMLOrJSONDecoder(bytes.NewBuffer(parsedNotebookTemplate), 8192).Decode(notebookCR) test.Expect(err).NotTo(gomega.HaveOccurred()) _, err = test.Client().Dynamic().Resource(notebookResource).Namespace(namespace.Name).Create(test.Ctx(), notebookCR, metav1.CreateOptions{}) test.Expect(err).NotTo(gomega.HaveOccurred()) diff --git a/test/odh/resources/custom-nb-small.yaml b/test/odh/resources/custom-nb-small.yaml index 95aaaf10..791a2d98 100644 --- a/test/odh/resources/custom-nb-small.yaml +++ b/test/odh/resources/custom-nb-small.yaml @@ -55,7 +55,7 @@ spec: - name: OCP_TOKEN value: {{.KubernetesBearerToken}} image: image-registry.openshift-image-registry.svc:5000/{{.OpenDataHubNamespace}}/{{.ImageStreamName}}:{{.ImageStreamTag}} - command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{.NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{.Namespace}} && sleep infinity"] + command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{.NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{.Namespace}} -p ray_image {{.RayImage}} && sleep infinity"] # args: ["pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/mcad.ipynb /opt/app-root/src/mcad-out.ipynb" ] imagePullPolicy: Always # livenessProbe: diff --git a/test/odh/resources/mnist.py b/test/odh/resources/mnist.py index d6a21194..e88e8fc9 100644 --- a/test/odh/resources/mnist.py +++ b/test/odh/resources/mnist.py @@ -15,6 +15,7 @@ import os import torch +import requests from pytorch_lightning import LightningModule, Trainer from pytorch_lightning.callbacks.progress import TQDMProgressBar from torch import nn @@ -32,6 +33,8 @@ print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) +MNIST_DATASET_URL = "{{.MnistDatasetURL}}" +print("MNIST_DATASET_URL: is ", MNIST_DATASET_URL) class LitMNIST(LightningModule): def __init__(self, data_dir=PATH_DATASETS, hidden_size=64, learning_rate=2e-4): @@ -110,8 +113,34 @@ def configure_optimizers(self): #################### def prepare_data(self): - # download - print("Downloading MNIST dataset...") + datasetFiles = [ + "t10k-images-idx3-ubyte", + "t10k-labels-idx1-ubyte", + "train-images-idx3-ubyte", + "train-labels-idx1-ubyte" + ] + + # Create required folder structure + downloadLocation = os.path.join(PATH_DATASETS, "MNIST", "raw") + os.makedirs(downloadLocation, exist_ok=True) + print(f"{downloadLocation} folder_path created!") + + for file in datasetFiles: + print(f"Downloading MNIST dataset {file}... to path : {downloadLocation}") + response = requests.get(f"{MNIST_DATASET_URL}{file}", stream=True) + filePath = os.path.join(downloadLocation, file) + + #to download dataset file + try: + if response.status_code == 200: + open(filePath, 'wb').write(response.content) + print(f"{file}: Downloaded and saved zipped file to path - {filePath}") + else: + print(f"Failed to download file {file}") + except Exception as e: + print(e) + print(f"Downloaded MNIST dataset to... {downloadLocation}") + MNIST(self.data_dir, train=True, download=True) MNIST(self.data_dir, train=False, download=True) diff --git a/test/odh/resources/mnist_ray_mini.ipynb b/test/odh/resources/mnist_ray_mini.ipynb index 38992cc7..0d8fcc53 100644 --- a/test/odh/resources/mnist_ray_mini.ipynb +++ b/test/odh/resources/mnist_ray_mini.ipynb @@ -27,7 +27,8 @@ "outputs": [], "source": [ "#parameters\n", - "namespace = \"default\"" + "namespace = \"default\"\n", + "ray_image = \"has to be specified\"" ] }, { @@ -40,7 +41,7 @@ "outputs": [], "source": [ "# Create our cluster and submit appwrapper\n", - "cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', head_cpus=1, head_memory=2, num_workers=1, min_cpus=1, max_cpus=1, min_memory=1, max_memory=2, num_gpus=0, instascale=False))" + "cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', head_cpus=1, head_memory=2, num_workers=1, min_cpus=1, max_cpus=1, min_memory=1, max_memory=2, num_gpus=0, instascale=False, image=ray_image))" ] }, { diff --git a/test/odh/resources/requirements.txt b/test/odh/resources/requirements.txt index 7266b064..cf9fb588 100644 --- a/test/odh/resources/requirements.txt +++ b/test/odh/resources/requirements.txt @@ -1,3 +1,5 @@ +{{.PipIndexUrl}} +{{.PipTrustedHost}} pytorch_lightning==1.5.10 ray_lightning torchmetrics==0.9.1 diff --git a/test/odh/template.go b/test/odh/template.go new file mode 100644 index 00000000..3ff4da17 --- /dev/null +++ b/test/odh/template.go @@ -0,0 +1,40 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package odh + +import ( + "bytes" + "html/template" + + "github.com/onsi/gomega" + "github.com/project-codeflare/codeflare-common/support" +) + +func ParseTemplate(t support.Test, inputTemplate []byte, props interface{}) []byte { + t.T().Helper() + + // Parse input template + parsedTemplate, err := template.New("template").Parse(string(inputTemplate)) + t.Expect(err).NotTo(gomega.HaveOccurred()) + + // Filter template and store results to the buffer + buffer := new(bytes.Buffer) + err = parsedTemplate.Execute(buffer, props) + t.Expect(err).NotTo(gomega.HaveOccurred()) + + return buffer.Bytes() +} From 8255d814fc752edcc6f089231f0f467a1748cdbd Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 8 Feb 2024 13:01:53 +0100 Subject: [PATCH 288/377] Update README.md with release instructions --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index de350d04..9a62419c 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,9 @@ For ODH tests additional environment variables are needed: 2. Once all jobs within the action are completed, verify that compatibility matrix in [README](https://github.com/project-codeflare/codeflare-operator/blob/main/README.md) was properly updated. 3. Verify that opened pull request to [OpenShift community operators repository](https://github.com/redhat-openshift-ecosystem/community-operators-prod) has proper content. 4. Once PR is merged, announce the new release in slack and mail lists, if any. -5. Update the Distributed Workloads component in ODH (also copy/update the compatibility matrix). This may require yaml and test updates depending on the release. Make sure to create a tag + release in the Distributed Workloads repository that matches the project-codeflare release version. -6. Update the readme/markdown/yaml in odh-manifests as required. +5. Release automation should open a PR with changes in [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Review the changes proposed by automation. If all the changes are correct then manually cherrypick all `CARRY` and `PATCH` commits from the current main branch, push the result to a dedicated branch and ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the `main` branch (branch protection has to be temporarily disabled). +6. Create a release branch on [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) for the next release if it doesn't exist yet. +7. Create a dedicated branch containing changes from [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Cherrypick all relevant changes available in [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) latest release branch which should be available also in the next release. Ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the release branch. ### Releases involving part of the stack From 1558401299be8c8a1b823468e0ff8cec67c80cd8 Mon Sep 17 00:00:00 2001 From: abdhumal Date: Fri, 2 Feb 2024 18:01:14 +0530 Subject: [PATCH 289/377] add an automated CI workflow to sync ODH-notebooks with Codeflare-SDK release --- .github/workflows/odh-notebooks-sync.yml | 116 ++++++++++++++++++ .../workflows/project-codeflare-release.yml | 17 +++ 2 files changed, 133 insertions(+) create mode 100644 .github/workflows/odh-notebooks-sync.yml diff --git a/.github/workflows/odh-notebooks-sync.yml b/.github/workflows/odh-notebooks-sync.yml new file mode 100644 index 00000000..789184c0 --- /dev/null +++ b/.github/workflows/odh-notebooks-sync.yml @@ -0,0 +1,116 @@ +# The aim of this GitHub workflow is to update the pipfile to sync with Codeflare-SDK release. +name: Sync with codeflare-sdk release +on: + workflow_dispatch: + inputs: + upstream-repository-organization: + required: true + description: "Owner of target upstream notebooks repository used to open a PR against" + default: "opendatahub-io" + + codeflare-repository-organization: + required: true + description: "Owner of origin notebooks repository used to open a PR" + default: "project-codeflare" + + codeflare_sdk_release_version: + required: true + description: "Provide version of the Codeflare-SDK release" + +env: + BRANCH_NAME: main + CODEFLARE_RELEASE_VERSION: ${{ github.event.inputs.codeflare_sdk_release_version }} + UPDATER_BRANCH: odh-sync-updater-${{ github.run_id }} + UPSTREAM_OWNER: ${{ github.event.inputs.upstream-repository-organization }} + REPO_OWNER: ${{ github.event.inputs.codeflare-repository-organization }} + REPO_NAME: notebooks + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.BRANCH_NAME }} + + - name: Clone repository and Sync + run: | + git clone https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git $REPO_NAME + cd $REPO_NAME + git remote add upstream https://github.com/$UPSTREAM_OWNER/$REPO_NAME.git + git config --global user.email "138894154+codeflare-machine-account@users.noreply.github.com" + git config --global user.name "codeflare-machine-account" + git remote -v + git pull upstream main && git push origin main + + - name: Setup Python environment + uses: actions/setup-python@v4 + with: + python-version: | + 3.8 + 3.9 + + - name: Install pipenv + run: pip install pipenv + + - name: Update Pipfiles in accordance with Codeflare-SDK latest release + run: | + # list all Pipfile paths having Codeflare-SDK listed + paths+=($(grep -rl 'codeflare-sdk = "~=.*"')) + # Extracting only directories from file paths, excluding a `.gitworkflow` directory + directories=() + exclude_directories=( + ".git/objects/pack" + ".github/workflows/", + ) + for path in "${paths[@]}"; do + current_dir=$(dirname "$path") + #Check if current_dir is not in exclude_directories list + if [[ ! "${exclude_directories[@]}" =~ "$current_dir" ]]; then + #Check if Pipfile exists in current_dir + if [ -f "$current_dir/Pipfile" ];then + directories+=("$current_dir") + fi + fi + done + # Remove duplicates + directories=($(echo "${directories[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + # Print the directories for verification + echo "Directories (Start updating Pipfile in these below directories in accordance with Codeflare-SDK latest release):" + for dir in "${directories[@]}"; do + echo "- $dir" + done + # iterate over the directories and update Pipfile + counter=0 + total=${#directories[@]} + for dir in "${directories[@]}"; do + counter=$((counter+1)) + echo "--Processing directory $counter '$dir' of total $total" + cd "$dir" && pipenv install codeflare-sdk~="${CODEFLARE_RELEASE_VERSION}" && pipenv --rm && cd - + echo "$((total-counter)) directories remaining.." + done + + - name: Push changes + run: | + cd $REPO_NAME + git add . && git status && git checkout -b ${{ env.UPDATER_BRANCH }} && \ + git commit -am "Updated notebooks via ${{ env.UPDATER_BRANCH }} GitHub action" --signoff && + git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git + git push origin ${{ env.UPDATER_BRANCH }} + + - name: Create Pull Request + run: | + gh pr create --repo $UPSTREAM_OWNER/$REPO_NAME \ + --title "$pr_title" \ + --body "$pr_body" \ + --head $REPO_OWNER:$UPDATER_BRANCH \ + --base $BRANCH_NAME + env: + pr_title: "[Digest Updater Action] Update notebook's pipfile to sync with Codeflare-SDK release" + pr_body: | + :rocket: This is an automated Pull Request. + + This PR updates the `Pipfile` to sync with latest Codeflare-SDK release. + + :exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.UPDATER_BRANCH }}` branch after merging the changes diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 77d0a0a1..7ed8899c 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -38,6 +38,10 @@ on: description: 'Owner of target community-operators-prod repository used to open a PR against' required: true default: 'redhat-openshift-ecosystem' + upstream-repository-organization: + description: 'Owner of target upstream notebooks repository used to open a PR against' + required: false + default: 'opendatahub-io' jobs: release-parameters: @@ -152,6 +156,19 @@ jobs: shell: bash if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} + - name: Sync ODH Notebooks + run: | + gh workflow run odh-notebooks-sync.yml \ + --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator \ + --ref ${{ github.ref }} \ + --field upstream-repository-organization=${{ github.event.inputs.upstream-repository-organization }} + --field codeflare-repository-organization=${{ github.event.inputs.codeflare-repository-organization }} \ + --field codeflare_sdk_release_version=${{ github.event.inputs.codeflare-sdk-version }} + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash + if: ${{ env.SDK_RELEASE_STATUS_CODE == '200' }} + release-codeflare-operator: needs: [release-mcad, release-instascale, release-codeflare-sdk] runs-on: ubuntu-latest From aa9c7ff410a516dce595591ede2e50ff62cd4e7a Mon Sep 17 00:00:00 2001 From: abdhumal Date: Fri, 9 Feb 2024 17:40:25 +0530 Subject: [PATCH 290/377] validate if codeflare-sdk version exists --- .github/workflows/odh-notebooks-sync.yml | 86 +++++++++++-------- .../workflows/project-codeflare-release.yml | 6 +- 2 files changed, 52 insertions(+), 40 deletions(-) diff --git a/.github/workflows/odh-notebooks-sync.yml b/.github/workflows/odh-notebooks-sync.yml index 789184c0..016608f2 100644 --- a/.github/workflows/odh-notebooks-sync.yml +++ b/.github/workflows/odh-notebooks-sync.yml @@ -51,45 +51,61 @@ jobs: 3.8 3.9 - - name: Install pipenv - run: pip install pipenv + - name: Install pipenv and pip-versions + run: pip install pipenv pip-versions - name: Update Pipfiles in accordance with Codeflare-SDK latest release run: | - # list all Pipfile paths having Codeflare-SDK listed - paths+=($(grep -rl 'codeflare-sdk = "~=.*"')) - # Extracting only directories from file paths, excluding a `.gitworkflow` directory - directories=() - exclude_directories=( - ".git/objects/pack" - ".github/workflows/", - ) - for path in "${paths[@]}"; do - current_dir=$(dirname "$path") - #Check if current_dir is not in exclude_directories list - if [[ ! "${exclude_directories[@]}" =~ "$current_dir" ]]; then - #Check if Pipfile exists in current_dir - if [ -f "$current_dir/Pipfile" ];then - directories+=("$current_dir") + package_name=codeflare-sdk + # Get the list of available versions for the package + if ! versions=$(pipenv run pip-versions list $package_name);then + echo "Failed to retrieve versions for $package_name" + exit 1 + fi + # Check if the desired version exists in the list + if echo "$versions" | grep -q "${CODEFLARE_RELEASE_VERSION}"; then + echo "Version ${CODEFLARE_RELEASE_VERSION} is available for $package_name" + # list all Pipfile paths having Codeflare-SDK listed + paths+=($(grep -rl "${package_name} = \"~=.*\"")) + # Extracting only directories from file paths, excluding a `.gitworkflow` directory + directories=() + exclude_directories=( + ".git/objects/pack" + ".github/workflows/", + ) + for path in "${paths[@]}"; do + current_dir=$(dirname "$path") + #Check if current_dir is not in exclude_directories list + if [[ ! "${exclude_directories[@]}" =~ "$current_dir" ]]; then + #Check if Pipfile exists in current_dir + if [ -f "$current_dir/Pipfile" ];then + directories+=("$current_dir") + fi fi - fi - done - # Remove duplicates - directories=($(echo "${directories[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) - # Print the directories for verification - echo "Directories (Start updating Pipfile in these below directories in accordance with Codeflare-SDK latest release):" - for dir in "${directories[@]}"; do - echo "- $dir" - done - # iterate over the directories and update Pipfile - counter=0 - total=${#directories[@]} - for dir in "${directories[@]}"; do - counter=$((counter+1)) - echo "--Processing directory $counter '$dir' of total $total" - cd "$dir" && pipenv install codeflare-sdk~="${CODEFLARE_RELEASE_VERSION}" && pipenv --rm && cd - - echo "$((total-counter)) directories remaining.." - done + done + # Remove duplicates + directories=($(echo "${directories[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + # Print the directories for verification + echo "Directories (Start updating Pipfile in these below directories in accordance with Codeflare-SDK latest release):" + for dir in "${directories[@]}"; do + echo "- $dir" + done + # iterate over the directories and update Pipfile + counter=0 + total=${#directories[@]} + for dir in "${directories[@]}"; do + counter=$((counter+1)) + echo "--Processing directory $counter '$dir' of total $total" + cd "$dir" && pipenv install ${package_name}~="${CODEFLARE_RELEASE_VERSION}" && pipenv --rm && cd - + echo "$((total-counter)) directories remaining.." + done + else + versions_list=$(echo "$versions" | tr '\n' ' ' | sed 's/, $//') + versions="${versions_list%,}" + echo "Version '${CODEFLARE_RELEASE_VERSION}' is not available for $package_name" + echo "Available versions for $package_name: $versions" + exit 1 + fi - name: Push changes run: | diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 7ed8899c..48a70ef4 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -38,10 +38,6 @@ on: description: 'Owner of target community-operators-prod repository used to open a PR against' required: true default: 'redhat-openshift-ecosystem' - upstream-repository-organization: - description: 'Owner of target upstream notebooks repository used to open a PR against' - required: false - default: 'opendatahub-io' jobs: release-parameters: @@ -161,7 +157,7 @@ jobs: gh workflow run odh-notebooks-sync.yml \ --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator \ --ref ${{ github.ref }} \ - --field upstream-repository-organization=${{ github.event.inputs.upstream-repository-organization }} + --field upstream-repository-organization=opendatahub-io --field codeflare-repository-organization=${{ github.event.inputs.codeflare-repository-organization }} \ --field codeflare_sdk_release_version=${{ github.event.inputs.codeflare-sdk-version }} env: From d0274d0afec620162730af032b0687845e1a0d1a Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 14 Feb 2024 15:06:08 +0100 Subject: [PATCH 291/377] Prepare Pytorch MNIST test image for disconnected testing --- .github/workflows/mnist-job-test-image.yml | 33 ++++ Makefile | 12 ++ test/pytorch_mnist_image/Dockerfile | 18 ++ test/pytorch_mnist_image/download_dataset.py | 21 +++ test/pytorch_mnist_image/entrypoint.sh | 3 + test/pytorch_mnist_image/mnist.py | 159 ++++++++++++++++++ .../mnist_pip_requirements.txt | 3 + 7 files changed, 249 insertions(+) create mode 100644 .github/workflows/mnist-job-test-image.yml create mode 100644 test/pytorch_mnist_image/Dockerfile create mode 100644 test/pytorch_mnist_image/download_dataset.py create mode 100755 test/pytorch_mnist_image/entrypoint.sh create mode 100644 test/pytorch_mnist_image/mnist.py create mode 100644 test/pytorch_mnist_image/mnist_pip_requirements.txt diff --git a/.github/workflows/mnist-job-test-image.yml b/.github/workflows/mnist-job-test-image.yml new file mode 100644 index 00000000..76810442 --- /dev/null +++ b/.github/workflows/mnist-job-test-image.yml @@ -0,0 +1,33 @@ +# This workflow will build the MNIST job test image and push it to the project-codeflare image registry + +name: MNIST Job Test Image + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - 'test/pytorch_mnist_image/**' + +jobs: + push: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.20 + + - name: Login to Quay.io + uses: redhat-actions/podman-login@v1 + with: + username: ${{ secrets.QUAY_ID }} + password: ${{ secrets.QUAY_TOKEN }} + registry: quay.io + + - name: Image Build and Push + run: | + make image-mnist-job-test-push diff --git a/Makefile b/Makefile index dccaea28..3b70333b 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,10 @@ ENVTEST_K8S_VERSION = 1.24.2 # used to build the manifests. ENV ?= default +# Image URL to build MNIST job test image +MNIST_JOB_TEST_VERSION ?= v0.0.2 +MNIST_JOB_TEST_IMG ?= $(IMAGE_ORG_BASE)/mnist-job-test:${MNIST_JOB_TEST_VERSION} + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -383,3 +387,11 @@ imports: openshift-goimports ## Organize imports in go files using openshift-goi .PHONY: verify-imports verify-imports: openshift-goimports ## Run import verifications. ./hack/verify-imports.sh $(OPENSHIFT-GOIMPORTS) + +.PHONY: image-mnist-job-test-build +image-mnist-job-test-build: ## Build container image with the MNIST job. + podman build -t ${MNIST_JOB_TEST_IMG} ./test/pytorch_mnist_image + +.PHONY: image-mnist-job-test-push +image-mnist-job-test-push: image-mnist-job-test-build ## Push container image with the MNIST job. + podman push ${MNIST_JOB_TEST_IMG} diff --git a/test/pytorch_mnist_image/Dockerfile b/test/pytorch_mnist_image/Dockerfile new file mode 100644 index 00000000..adbbd7cb --- /dev/null +++ b/test/pytorch_mnist_image/Dockerfile @@ -0,0 +1,18 @@ +# Build the manager binary +FROM pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime + +WORKDIR /test +COPY entrypoint.sh entrypoint.sh + +# Install MNIST requirements +COPY mnist_pip_requirements.txt requirements.txt +RUN pip install --requirement requirements.txt + +# Prepare MNIST script +COPY mnist.py mnist.py +COPY download_dataset.py download_dataset.py +RUN torchrun download_dataset.py + +USER 65532:65532 +WORKDIR /workdir +ENTRYPOINT ["/test/entrypoint.sh"] diff --git a/test/pytorch_mnist_image/download_dataset.py b/test/pytorch_mnist_image/download_dataset.py new file mode 100644 index 00000000..698f68f6 --- /dev/null +++ b/test/pytorch_mnist_image/download_dataset.py @@ -0,0 +1,21 @@ +# Copyright 2022 IBM, Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from torchvision.datasets import MNIST + +PATH_DATASETS = os.environ.get("PATH_DATASETS", ".") +MNIST(PATH_DATASETS, train=True, download=True) +MNIST(PATH_DATASETS, train=False, download=True) diff --git a/test/pytorch_mnist_image/entrypoint.sh b/test/pytorch_mnist_image/entrypoint.sh new file mode 100755 index 00000000..7487b377 --- /dev/null +++ b/test/pytorch_mnist_image/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +torchrun /test/mnist.py diff --git a/test/pytorch_mnist_image/mnist.py b/test/pytorch_mnist_image/mnist.py new file mode 100644 index 00000000..134d5618 --- /dev/null +++ b/test/pytorch_mnist_image/mnist.py @@ -0,0 +1,159 @@ +# Copyright 2022 IBM, Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch +import requests +from pytorch_lightning import LightningModule, Trainer +from pytorch_lightning.callbacks.progress import TQDMProgressBar +from torch import nn +from torch.nn import functional as F +from torch.utils.data import DataLoader, random_split +from torchmetrics import Accuracy +from torchvision import transforms +from torchvision.datasets import MNIST + +PATH_WORKDIR = os.environ.get("PATH_WORKDIR", ".") +PATH_DATASETS = os.environ.get("PATH_DATASETS", "/test") +BATCH_SIZE = 256 if torch.cuda.is_available() else 64 +# %% + +print("prior to running the trainer") +print("MASTER_ADDR: is ", os.getenv("MASTER_ADDR")) +print("MASTER_PORT: is ", os.getenv("MASTER_PORT")) + +class LitMNIST(LightningModule): + def __init__(self, data_dir=PATH_WORKDIR, hidden_size=64, learning_rate=2e-4): + + super().__init__() + + # Set our init args as class attributes + self.data_dir = data_dir + self.hidden_size = hidden_size + self.learning_rate = learning_rate + + # Hardcode some dataset specific attributes + self.num_classes = 10 + self.dims = (1, 28, 28) + channels, width, height = self.dims + self.transform = transforms.Compose( + [ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)), + ] + ) + + # Define PyTorch model + self.model = nn.Sequential( + nn.Flatten(), + nn.Linear(channels * width * height, hidden_size), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden_size, hidden_size), + nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(hidden_size, self.num_classes), + ) + + self.val_accuracy = Accuracy() + self.test_accuracy = Accuracy() + + def forward(self, x): + x = self.model(x) + return F.log_softmax(x, dim=1) + + def training_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + return loss + + def validation_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + preds = torch.argmax(logits, dim=1) + self.val_accuracy.update(preds, y) + + # Calling self.log will surface up scalars for you in TensorBoard + self.log("val_loss", loss, prog_bar=True) + self.log("val_acc", self.val_accuracy, prog_bar=True) + + def test_step(self, batch, batch_idx): + x, y = batch + logits = self(x) + loss = F.nll_loss(logits, y) + preds = torch.argmax(logits, dim=1) + self.test_accuracy.update(preds, y) + + # Calling self.log will surface up scalars for you in TensorBoard + self.log("test_loss", loss, prog_bar=True) + self.log("test_acc", self.test_accuracy, prog_bar=True) + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) + return optimizer + + #################### + # DATA RELATED HOOKS + #################### + + def prepare_data(self): + MNIST(PATH_DATASETS, train=True, download=True) + MNIST(PATH_DATASETS, train=False, download=True) + + def setup(self, stage=None): + + # Assign train/val datasets for use in dataloaders + if stage == "fit" or stage is None: + mnist_full = MNIST(PATH_DATASETS, train=True, transform=self.transform) + self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000]) + + # Assign test dataset for use in dataloader(s) + if stage == "test" or stage is None: + self.mnist_test = MNIST( + PATH_DATASETS, train=False, transform=self.transform + ) + + def train_dataloader(self): + return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) + + def val_dataloader(self): + return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) + + def test_dataloader(self): + return DataLoader(self.mnist_test, batch_size=BATCH_SIZE) + + +# Init DataLoader from MNIST Dataset + +model = LitMNIST() + +print("GROUP: ", int(os.environ.get("GROUP_WORLD_SIZE", 1))) +print("LOCAL: ", int(os.environ.get("LOCAL_WORLD_SIZE", 1))) + +# Initialize a trainer +trainer = Trainer( + accelerator="auto", + # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs + max_epochs=5, + callbacks=[TQDMProgressBar(refresh_rate=20)], + num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), + devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), + strategy="ddp", +) + +# Train the model ⚡ +trainer.fit(model) diff --git a/test/pytorch_mnist_image/mnist_pip_requirements.txt b/test/pytorch_mnist_image/mnist_pip_requirements.txt new file mode 100644 index 00000000..87edeef2 --- /dev/null +++ b/test/pytorch_mnist_image/mnist_pip_requirements.txt @@ -0,0 +1,3 @@ +pytorch_lightning==1.5.10 +torchmetrics==0.9.1 +torchvision==0.12.0 From 705ee164fb290eec9a165817ac8259fcc4b44ceb Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 15 Feb 2024 14:52:58 +0100 Subject: [PATCH 292/377] Update deprecated GitHub actions --- .github/workflows/e2e_tests.yaml | 8 ++++---- .github/workflows/mnist-job-test-image.yml | 15 ++++++++------ .github/workflows/olm_tests.yaml | 8 ++++---- .github/workflows/operator-image.yml | 17 +++++++++------- .github/workflows/precommit.yml | 6 +++--- .github/workflows/tag-and-build.yml | 21 +++++++++++--------- .github/workflows/unit_tests.yml | 6 +++--- .github/workflows/verify_generated_files.yml | 8 ++++---- 8 files changed, 49 insertions(+), 40 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 0786c290..0fc1ec88 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -31,19 +31,19 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Checkout common repo code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: 'project-codeflare/codeflare-common' ref: 'main' path: 'common' - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 @@ -99,7 +99,7 @@ jobs: output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} - name: Upload logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() && steps.deploy.outcome == 'success' with: name: logs diff --git a/.github/workflows/mnist-job-test-image.yml b/.github/workflows/mnist-job-test-image.yml index 76810442..4323bd5d 100644 --- a/.github/workflows/mnist-job-test-image.yml +++ b/.github/workflows/mnist-job-test-image.yml @@ -17,17 +17,20 @@ jobs: - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 - name: Login to Quay.io - uses: redhat-actions/podman-login@v1 - with: - username: ${{ secrets.QUAY_ID }} - password: ${{ secrets.QUAY_TOKEN }} - registry: quay.io + id: podman-login-quay + run: | + podman login --username ${{ secrets.QUAY_ID }} --password ${{ secrets.QUAY_TOKEN }} quay.io - name: Image Build and Push run: | make image-mnist-job-test-push + + - name: Logout from Quay.io + if: always() && steps.podman-login-quay.outcome == 'success' + run: | + podman logout quay.io diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 6889604b..aa8a3b17 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -30,19 +30,19 @@ jobs: CODEFLARE_TEST_TIMEOUT_LONG: "10m" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 # fetching also previous commits to get tags - name: Checkout common repo code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: 'project-codeflare/codeflare-common' ref: 'main' path: 'common' - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 @@ -192,7 +192,7 @@ jobs: output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} - name: Upload logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() && steps.deploy.outcome == 'success' with: name: logs diff --git a/.github/workflows/operator-image.yml b/.github/workflows/operator-image.yml index 0c155be1..8c30a8c1 100644 --- a/.github/workflows/operator-image.yml +++ b/.github/workflows/operator-image.yml @@ -17,22 +17,25 @@ jobs: push: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 - name: Login to Quay.io - uses: redhat-actions/podman-login@v1 - with: - username: ${{ secrets.QUAY_ID }} - password: ${{ secrets.QUAY_TOKEN }} - registry: quay.io + id: podman-login-quay + run: | + podman login --username ${{ secrets.QUAY_ID }} --password ${{ secrets.QUAY_TOKEN }} quay.io - name: Image Build run: | make build make image-build -e IMG=quay.io/project-codeflare/codeflare-operator:dev make image-push -e IMG=quay.io/project-codeflare/codeflare-operator:dev + + - name: Logout from Quay.io + if: always() && steps.podman-login-quay.outcome == 'success' + run: | + podman logout quay.io diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 13c4c7fc..2ca5869c 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -21,15 +21,15 @@ jobs: volumes: - /cache steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 - name: Activate cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /cache key: ${{ runner.os }}-cache-${{ hashFiles('**/go.sum', '.pre-commit-config.yaml') }} diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index f04d43d6..9dc290c4 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -59,10 +59,10 @@ jobs: PR_BRANCH_NAME: adjustments-release-${{ github.event.inputs.version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 @@ -79,7 +79,7 @@ jobs: GITHUB_TOKEN: ${{ github.TOKEN }} - name: Activate cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /cache key: ${{ runner.os }}-cache-${{ hashFiles('**/go.sum', '.pre-commit-config.yaml') }} @@ -108,11 +108,9 @@ jobs: shell: bash - name: Login to Quay.io - uses: redhat-actions/podman-login@v1 - with: - username: ${{ secrets.QUAY_ID }} - password: ${{ secrets.QUAY_TOKEN }} - registry: quay.io + id: podman-login-quay + run: | + podman login --username ${{ secrets.QUAY_ID }} --password ${{ secrets.QUAY_TOKEN }} quay.io - name: Align go.mod and go.sum dependencies for released components run: | @@ -140,7 +138,7 @@ jobs: - name: Commit changes in the code back to repository id: create-pr-branch - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@v5 with: commit_message: Update dependency versions for release ${{ github.event.inputs.version }} file_pattern: 'README.md *.yaml Makefile go.mod go.sum *.env' @@ -199,3 +197,8 @@ jobs: GH_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} + + - name: Logout from Quay.io + if: always() && steps.podman-login-quay.outcome == 'success' + run: | + podman logout quay.io diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 39ab0ef0..747cee03 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -22,15 +22,15 @@ jobs: volumes: - /cache steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 - name: Activate cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /cache key: ${{ runner.os }}-cache-${{ hashFiles('**/go.sum', '.pre-commit-config.yaml') }} diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml index e2fa269c..9027c1aa 100644 --- a/.github/workflows/verify_generated_files.yml +++ b/.github/workflows/verify_generated_files.yml @@ -20,9 +20,9 @@ jobs: verify-imports: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 - name: Verify that imports are organized @@ -31,9 +31,9 @@ jobs: verify-manifests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: v1.20 - name: Verify that the latest WebhookConfigurations, ClusterRoles, and CustomResourceDefinitions have been generated From ac483fdcc2f8f0d3f3b2b4f2d57b507ce583cad8 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 19 Feb 2024 15:33:12 +0100 Subject: [PATCH 293/377] e2e tests: Use test image from quay.io/project-codeflare/demo-images --- test/odh/resources/mnist_mcad_mini.ipynb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/odh/resources/mnist_mcad_mini.ipynb b/test/odh/resources/mnist_mcad_mini.ipynb index 0b53324a..b32a95ab 100644 --- a/test/odh/resources/mnist_mcad_mini.ipynb +++ b/test/odh/resources/mnist_mcad_mini.ipynb @@ -19,7 +19,9 @@ "execution_count": null, "id": "47ca5c15", "metadata": { - "tags": ["parameters"] + "tags": [ + "parameters" + ] }, "outputs": [], "source": [ @@ -34,7 +36,7 @@ "metadata": {}, "outputs": [], "source": [ - "job = DDPJobDefinition(name=\"mnistjob\", script=\"mnist.py\", scheduler_args={\"namespace\": namespace}, j=\"1x1\", gpu=0, cpu=1, memMB=2000, image=\"quay.io/project-codeflare/mnist-job-test:v0.0.1\").submit()" + "job = DDPJobDefinition(name=\"mnistjob\", script=\"/test/mnist.py\", scheduler_args={\"namespace\": namespace}, j=\"1x1\", gpu=0, cpu=1, memMB=2000, image=\"quay.io/project-codeflare/demo-images:pytorch-mnist-v0.0.1\").submit()" ] }, { From 10bf0dd05f803c59cb565b24f46033697569d2c8 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 20 Feb 2024 15:54:59 +0000 Subject: [PATCH 294/377] Fix MCAD release --- .github/workflows/project-codeflare-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 48a70ef4..b7abf8e8 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -70,7 +70,7 @@ jobs: - name: Release MCAD run: | - gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref ${{ github.ref }} --field tag=${{ github.event.inputs.mcad-version }} + gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref go-1.20 --field tag=${{ github.event.inputs.mcad-version }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash From 2e9384965b268416fa6ff9880bbe479d42a6718c Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 20 Feb 2024 17:08:16 +0100 Subject: [PATCH 295/377] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9a62419c..aadb28f8 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ For ODH tests additional environment variables are needed: 5. Release automation should open a PR with changes in [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Review the changes proposed by automation. If all the changes are correct then manually cherrypick all `CARRY` and `PATCH` commits from the current main branch, push the result to a dedicated branch and ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the `main` branch (branch protection has to be temporarily disabled). 6. Create a release branch on [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) for the next release if it doesn't exist yet. 7. Create a dedicated branch containing changes from [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Cherrypick all relevant changes available in [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) latest release branch which should be available also in the next release. Ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the release branch. +8. Make sure that release automation created a PR updating CodeFlare SDK version in [ODH Notebooks repository](https://github.com/opendatahub-io/notebooks). Make sure the PR gets merged. ### Releases involving part of the stack From dfd6c619452f03fa2f9c1011f7b9840b3e4b499c Mon Sep 17 00:00:00 2001 From: dimakis Date: Tue, 20 Feb 2024 23:06:28 +0000 Subject: [PATCH 296/377] Update dependency versions for release v1.2.0 --- Makefile | 2 +- README.md | 6 +++--- config/crd/mcad/kustomization.yaml | 2 +- config/manager/params.env | 2 +- go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 3b70333b..e1419e1a 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ INSTASCALE_VERSION ?= v0.4.0 INSTASCALE_REPO ?= github.com/project-codeflare/instascale # MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.39.0 +MCAD_VERSION ?= v1.40.0 MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher # Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} diff --git a/README.md b/README.md index aadb28f8..73bb5549 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.1.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.1.0) | -| Multi-Cluster App Dispatcher | [v1.39.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.39.0) | -| CodeFlare-SDK | [v0.13.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.13.0) | +| CodeFlare Operator | [v1.2.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.2.0) | +| Multi-Cluster App Dispatcher | [v1.40.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.40.0) | +| CodeFlare-SDK | [v0.14.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.14.0) | | InstaScale | [v0.4.0](https://github.com/project-codeflare/instascale/releases/tag/v0.4.0) | | KubeRay | [v1.0.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0) | diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index 0e4ebfa7..bbf6141c 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.39.0 +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.40.0 diff --git a/config/manager/params.env b/config/manager/params.env index ace2f2f2..c4c8b45f 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1 +1 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.1.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.2.0 diff --git a/go.mod b/go.mod index 252653db..7f846b2b 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 github.com/project-codeflare/instascale v0.4.0 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 + github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0 github.com/ray-project/kuberay/ray-operator v1.0.0 go.uber.org/zap v1.26.0 k8s.io/api v0.27.8 diff --git a/go.sum b/go.sum index b36099d6..60345cd5 100644 --- a/go.sum +++ b/go.sum @@ -388,8 +388,8 @@ github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo= github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0 h1:zoS7pEAWK6eGELPCIIHB3W8Zb/a27Rf55ChYso7EV3o= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.39.0/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0 h1:IkTmd/W/zxcsC5s4EbnW74PFpkQVEiTc/8rWWwFw0Ok= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= From 49b2f9da99305a9da7f5b12933509befe492156b Mon Sep 17 00:00:00 2001 From: abhijeet-dhumal Date: Wed, 21 Feb 2024 15:13:59 +0530 Subject: [PATCH 297/377] fixed incorrect logic in Codeflare-SDK release sync with notebooks --- .github/workflows/project-codeflare-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index b7abf8e8..d5e0fe06 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -163,7 +163,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash - if: ${{ env.SDK_RELEASE_STATUS_CODE == '200' }} + if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} release-codeflare-operator: needs: [release-mcad, release-instascale, release-codeflare-sdk] From 6b5acfcd315ba71813ebe98ef6c996a5501092b8 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 20 Feb 2024 16:32:12 +0000 Subject: [PATCH 298/377] Display input for KubeRay parameter and check for existence --- .github/workflows/project-codeflare-release.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index d5e0fe06..6e2d7574 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -51,12 +51,14 @@ jobs: echo "MCAD Version: ${{ github.event.inputs.mcad-version }}" echo "CodeFlare SDK Version: ${{ github.event.inputs.codeflare-sdk-version }}" echo "InstaScale Version: ${{ github.event.inputs.instascale-version }}" + echo "Tested KubeRay Version: ${{ github.event.inputs.kuberay-version }}" echo "Is Stable: ${{ github.event.inputs.is-stable }}" echo "CodeFlare Repository Organization: ${{ github.event.inputs.codeflare-repository-organization }}" echo "Quay Organization: ${{ github.event.inputs.quay-organization }}" echo "Community Operators Prod Organization: ${{ github.event.inputs.community-operators-prod-organization }}" release-mcad: + needs: check-kuberay-version runs-on: ubuntu-latest steps: @@ -120,6 +122,7 @@ jobs: if: ${{ env.INSTASCALE_RELEASE_STATUS_CODE != '200' }} release-codeflare-sdk: + needs: check-kuberay-version runs-on: ubuntu-latest steps: @@ -186,3 +189,17 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash + + check-kuberay-version: + runs-on: ubuntu-latest + + steps: + - name: Check if KubeRay release exists + run: | + status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/ray-project/kuberay/releases/tag/${{ github.event.inputs.kuberay-version }}) + if [[ "$status_code" == "200" ]]; then + echo "KubeRay release with version ${{ github.event.inputs.kuberay-version }} exists and available to use." + else + echo "KubeRay release with version ${{ github.event.inputs.kuberay-version }} does not exist. Please select an existing version." + exit 1 + fi From fa8863602903a319605529a87b8561082eda502f Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Fri, 9 Feb 2024 16:25:16 +0000 Subject: [PATCH 299/377] add: update machinepool tests to identify machines based on label --- test/e2e/instascale_machinepool_test.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 69fa037d..64e53f8a 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -17,6 +17,7 @@ limitations under the License. package e2e import ( + "fmt" "testing" . "github.com/onsi/gomega" @@ -49,14 +50,15 @@ func TestInstascaleMachinePool(t *testing.T) { connection := CreateOCMConnection(test) defer connection.Close() - // check existing cluster machine pool resources - // look for machine pool with aw name - expect not to find it - test.Expect(GetMachinePools(test, connection)). - ShouldNot(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) - // Setup batch job and AppWrapper aw := instaScaleJobAppWrapper(test, namespace, cm) + expectedLabel := fmt.Sprintf("%s-%s", aw.Name, aw.Namespace) + // check existing cluster machine pool resources + // look for a machine pool with a label key equal to aw.name-aw.namespace - expect NOT to find it + test.Expect(GetMachinePools(test, connection)). + ShouldNot(ContainElement(WithTransform(MachinePoolLabels, HaveKey(expectedLabel)))) + // apply AppWrapper to cluster _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) @@ -66,15 +68,15 @@ func TestInstascaleMachinePool(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - // look for machine pool with aw name - expect to find it + // look for a machine pool with a label key equal to aw.name-aw.namespace - expect to find it test.Eventually(MachinePools(test, connection), TestTimeoutLong). - Should(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) + Should(ContainElement(WithTransform(MachinePoolLabels, HaveKey(expectedLabel)))) test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - // look for machine pool with aw name - expect not to find it + // look for a machine pool with a label key equal to aw.name-aw.namespace - expect NOT to find it test.Eventually(MachinePools(test, connection), TestTimeoutLong). - ShouldNot(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) + ShouldNot(ContainElement(WithTransform(MachinePoolLabels, HaveKey(expectedLabel)))) } From 8aeaefc831335c978a4109d5cdf6654664855063 Mon Sep 17 00:00:00 2001 From: abhijeet-dhumal Date: Tue, 27 Feb 2024 16:35:46 +0530 Subject: [PATCH 300/377] remove Sync ODH-notebooks CI workflow to add this step in Codeflare-SDK repository --- .github/workflows/odh-notebooks-sync.yml | 132 ------------------ .../workflows/project-codeflare-release.yml | 13 -- 2 files changed, 145 deletions(-) delete mode 100644 .github/workflows/odh-notebooks-sync.yml diff --git a/.github/workflows/odh-notebooks-sync.yml b/.github/workflows/odh-notebooks-sync.yml deleted file mode 100644 index 016608f2..00000000 --- a/.github/workflows/odh-notebooks-sync.yml +++ /dev/null @@ -1,132 +0,0 @@ -# The aim of this GitHub workflow is to update the pipfile to sync with Codeflare-SDK release. -name: Sync with codeflare-sdk release -on: - workflow_dispatch: - inputs: - upstream-repository-organization: - required: true - description: "Owner of target upstream notebooks repository used to open a PR against" - default: "opendatahub-io" - - codeflare-repository-organization: - required: true - description: "Owner of origin notebooks repository used to open a PR" - default: "project-codeflare" - - codeflare_sdk_release_version: - required: true - description: "Provide version of the Codeflare-SDK release" - -env: - BRANCH_NAME: main - CODEFLARE_RELEASE_VERSION: ${{ github.event.inputs.codeflare_sdk_release_version }} - UPDATER_BRANCH: odh-sync-updater-${{ github.run_id }} - UPSTREAM_OWNER: ${{ github.event.inputs.upstream-repository-organization }} - REPO_OWNER: ${{ github.event.inputs.codeflare-repository-organization }} - REPO_NAME: notebooks - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ env.BRANCH_NAME }} - - - name: Clone repository and Sync - run: | - git clone https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git $REPO_NAME - cd $REPO_NAME - git remote add upstream https://github.com/$UPSTREAM_OWNER/$REPO_NAME.git - git config --global user.email "138894154+codeflare-machine-account@users.noreply.github.com" - git config --global user.name "codeflare-machine-account" - git remote -v - git pull upstream main && git push origin main - - - name: Setup Python environment - uses: actions/setup-python@v4 - with: - python-version: | - 3.8 - 3.9 - - - name: Install pipenv and pip-versions - run: pip install pipenv pip-versions - - - name: Update Pipfiles in accordance with Codeflare-SDK latest release - run: | - package_name=codeflare-sdk - # Get the list of available versions for the package - if ! versions=$(pipenv run pip-versions list $package_name);then - echo "Failed to retrieve versions for $package_name" - exit 1 - fi - # Check if the desired version exists in the list - if echo "$versions" | grep -q "${CODEFLARE_RELEASE_VERSION}"; then - echo "Version ${CODEFLARE_RELEASE_VERSION} is available for $package_name" - # list all Pipfile paths having Codeflare-SDK listed - paths+=($(grep -rl "${package_name} = \"~=.*\"")) - # Extracting only directories from file paths, excluding a `.gitworkflow` directory - directories=() - exclude_directories=( - ".git/objects/pack" - ".github/workflows/", - ) - for path in "${paths[@]}"; do - current_dir=$(dirname "$path") - #Check if current_dir is not in exclude_directories list - if [[ ! "${exclude_directories[@]}" =~ "$current_dir" ]]; then - #Check if Pipfile exists in current_dir - if [ -f "$current_dir/Pipfile" ];then - directories+=("$current_dir") - fi - fi - done - # Remove duplicates - directories=($(echo "${directories[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) - # Print the directories for verification - echo "Directories (Start updating Pipfile in these below directories in accordance with Codeflare-SDK latest release):" - for dir in "${directories[@]}"; do - echo "- $dir" - done - # iterate over the directories and update Pipfile - counter=0 - total=${#directories[@]} - for dir in "${directories[@]}"; do - counter=$((counter+1)) - echo "--Processing directory $counter '$dir' of total $total" - cd "$dir" && pipenv install ${package_name}~="${CODEFLARE_RELEASE_VERSION}" && pipenv --rm && cd - - echo "$((total-counter)) directories remaining.." - done - else - versions_list=$(echo "$versions" | tr '\n' ' ' | sed 's/, $//') - versions="${versions_list%,}" - echo "Version '${CODEFLARE_RELEASE_VERSION}' is not available for $package_name" - echo "Available versions for $package_name: $versions" - exit 1 - fi - - - name: Push changes - run: | - cd $REPO_NAME - git add . && git status && git checkout -b ${{ env.UPDATER_BRANCH }} && \ - git commit -am "Updated notebooks via ${{ env.UPDATER_BRANCH }} GitHub action" --signoff && - git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/$REPO_OWNER/$REPO_NAME.git - git push origin ${{ env.UPDATER_BRANCH }} - - - name: Create Pull Request - run: | - gh pr create --repo $UPSTREAM_OWNER/$REPO_NAME \ - --title "$pr_title" \ - --body "$pr_body" \ - --head $REPO_OWNER:$UPDATER_BRANCH \ - --base $BRANCH_NAME - env: - pr_title: "[Digest Updater Action] Update notebook's pipfile to sync with Codeflare-SDK release" - pr_body: | - :rocket: This is an automated Pull Request. - - This PR updates the `Pipfile` to sync with latest Codeflare-SDK release. - - :exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.UPDATER_BRANCH }}` branch after merging the changes diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 6e2d7574..50fc26ae 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -155,19 +155,6 @@ jobs: shell: bash if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} - - name: Sync ODH Notebooks - run: | - gh workflow run odh-notebooks-sync.yml \ - --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator \ - --ref ${{ github.ref }} \ - --field upstream-repository-organization=opendatahub-io - --field codeflare-repository-organization=${{ github.event.inputs.codeflare-repository-organization }} \ - --field codeflare_sdk_release_version=${{ github.event.inputs.codeflare-sdk-version }} - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - shell: bash - if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} - release-codeflare-operator: needs: [release-mcad, release-instascale, release-codeflare-sdk] runs-on: ubuntu-latest From 7a9b5bb40ef79aa0788d0279d935895e57ab7c0a Mon Sep 17 00:00:00 2001 From: Jiri Petrlik Date: Tue, 27 Feb 2024 16:24:39 +0100 Subject: [PATCH 301/377] RHOAIENG-3771 - Reduce execution time of E2E tests By reducing number of epochs and number of training samples in each epoch it was possible to reduce test execution time from more than 10 minutes to less than 2 minutes. --- test/e2e/mnist.py | 7 ++++--- test/odh/resources/mnist.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/test/e2e/mnist.py b/test/e2e/mnist.py index 4cfe0b43..5ac26665 100644 --- a/test/e2e/mnist.py +++ b/test/e2e/mnist.py @@ -20,7 +20,7 @@ from pytorch_lightning.callbacks.progress import TQDMProgressBar from torch import nn from torch.nn import functional as F -from torch.utils.data import DataLoader, random_split +from torch.utils.data import DataLoader, random_split, RandomSampler from torchmetrics import Accuracy from torchvision import transforms from torchvision.datasets import MNIST @@ -158,7 +158,7 @@ def setup(self, stage=None): ) def train_dataloader(self): - return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) + return DataLoader(self.mnist_train, batch_size=BATCH_SIZE, sampler=RandomSampler(self.mnist_train, num_samples=1000)) def val_dataloader(self): return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) @@ -178,10 +178,11 @@ def test_dataloader(self): trainer = Trainer( accelerator="auto", # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs - max_epochs=5, + max_epochs=3, callbacks=[TQDMProgressBar(refresh_rate=20)], num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), + replace_sampler_ddp=False, strategy="ddp", ) diff --git a/test/odh/resources/mnist.py b/test/odh/resources/mnist.py index e88e8fc9..85d420f4 100644 --- a/test/odh/resources/mnist.py +++ b/test/odh/resources/mnist.py @@ -20,7 +20,7 @@ from pytorch_lightning.callbacks.progress import TQDMProgressBar from torch import nn from torch.nn import functional as F -from torch.utils.data import DataLoader, random_split +from torch.utils.data import DataLoader, random_split, RandomSampler from torchmetrics import Accuracy from torchvision import transforms from torchvision.datasets import MNIST @@ -158,7 +158,7 @@ def setup(self, stage=None): ) def train_dataloader(self): - return DataLoader(self.mnist_train, batch_size=BATCH_SIZE) + return DataLoader(self.mnist_train, batch_size=BATCH_SIZE, sampler=RandomSampler(self.mnist_train, num_samples=1000)) def val_dataloader(self): return DataLoader(self.mnist_val, batch_size=BATCH_SIZE) @@ -178,10 +178,11 @@ def test_dataloader(self): trainer = Trainer( accelerator="auto", # devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs - max_epochs=2, + max_epochs=3, callbacks=[TQDMProgressBar(refresh_rate=20)], num_nodes=int(os.environ.get("GROUP_WORLD_SIZE", 1)), devices=int(os.environ.get("LOCAL_WORLD_SIZE", 1)), + replace_sampler_ddp=False, strategy="ddp", ) From bb2460797387b02d918d79c13360801b7de8434d Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 28 Feb 2024 14:14:48 +0100 Subject: [PATCH 302/377] Update README.md with ODH/CFO build instructions --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 73bb5549..4f9cd024 100644 --- a/README.md +++ b/README.md @@ -108,9 +108,10 @@ For ODH tests additional environment variables are needed: 3. Verify that opened pull request to [OpenShift community operators repository](https://github.com/redhat-openshift-ecosystem/community-operators-prod) has proper content. 4. Once PR is merged, announce the new release in slack and mail lists, if any. 5. Release automation should open a PR with changes in [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Review the changes proposed by automation. If all the changes are correct then manually cherrypick all `CARRY` and `PATCH` commits from the current main branch, push the result to a dedicated branch and ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the `main` branch (branch protection has to be temporarily disabled). -6. Create a release branch on [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) for the next release if it doesn't exist yet. -7. Create a dedicated branch containing changes from [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Cherrypick all relevant changes available in [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) latest release branch which should be available also in the next release. Ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the release branch. -8. Make sure that release automation created a PR updating CodeFlare SDK version in [ODH Notebooks repository](https://github.com/opendatahub-io/notebooks). Make sure the PR gets merged. +6. Build ODH/CFO image by triggering [Build and Push action](https://github.com/opendatahub-io/codeflare-operator/actions/workflows/build-and-push.yaml) +7. Create a release branch on [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) for the next release if it doesn't exist yet. +8. Create a dedicated branch containing changes from [ODH CodeFlare operator repo](https://github.com/opendatahub-io/codeflare-operator). Cherrypick all relevant changes available in [Red Hat CodeFlare operator repo](https://github.com/red-hat-data-services/codeflare-operator) latest release branch which should be available also in the next release. Ask on Slack channel for review of the result branch content. Once agreed then push the changes directly to the release branch. +9. Make sure that release automation created a PR updating CodeFlare SDK version in [ODH Notebooks repository](https://github.com/opendatahub-io/notebooks). Make sure the PR gets merged. ### Releases involving part of the stack From 7419586f1a169fe956f3dd476f88ca3a4a807138 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 8 Mar 2024 17:17:58 +0100 Subject: [PATCH 303/377] e2e tests: Fix finishing condition in ODH e2e test --- test/odh/resources/mnist_mcad_mini.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/odh/resources/mnist_mcad_mini.ipynb b/test/odh/resources/mnist_mcad_mini.ipynb index b32a95ab..341eb12c 100644 --- a/test/odh/resources/mnist_mcad_mini.ipynb +++ b/test/odh/resources/mnist_mcad_mini.ipynb @@ -50,7 +50,7 @@ "while not finished:\n", " sleep(1)\n", " try:\n", - " finished = (\"Epoch 4: 100%\" in job.logs())\n", + " finished = (\"Epoch 2: 100%\" in job.logs())\n", " except:\n", " finished = False" ] From 2b5d9b838ad2141f15e97ccfd8b705cf82cf395c Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 25 Mar 2024 10:27:48 +0000 Subject: [PATCH 304/377] Fix e2e test - invalid go version error --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e1419e1a..d2cd22d0 100644 --- a/Makefile +++ b/Makefile @@ -267,7 +267,7 @@ $(YQ): $(LOCALBIN) .PHONY: envtest envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) - test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.17 .PHONY: openshift-goimports openshift-goimports: $(OPENSHIFT-GOIMPORTS) ## Download openshift-goimports locally if necessary. From 91f8bf3aeb8fb2a2836a8c51cbbeb9f5e8d300ad Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 25 Jan 2024 13:48:14 -0500 Subject: [PATCH 305/377] add raycluster controller to CFO Signed-off-by: Kevin --- Dockerfile | 1 + PROJECT | 10 + config/rbac/kustomization.yaml | 2 + config/rbac/mcad_manager_role.yaml | 223 ++++++++++++++ config/rbac/mcad_manager_role_binding.yaml | 12 + config/rbac/role.yaml | 179 +---------- controllers/raycluster_controller.go | 338 +++++++++++++++++++++ controllers/raycluster_controller_test.go | 222 ++++++++++++++ controllers/suite_test.go | 135 ++++++++ go.mod | 8 +- go.sum | 10 +- main.go | 33 ++ 12 files changed, 1001 insertions(+), 172 deletions(-) create mode 100644 config/rbac/mcad_manager_role.yaml create mode 100644 config/rbac/mcad_manager_role_binding.yaml create mode 100644 controllers/raycluster_controller.go create mode 100644 controllers/raycluster_controller_test.go create mode 100644 controllers/suite_test.go diff --git a/Dockerfile b/Dockerfile index f5c51c3f..26f4c23d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN go mod download # Copy the Go sources COPY main.go main.go COPY pkg/ pkg/ +COPY controllers/ controllers/ # Build USER root diff --git a/PROJECT b/PROJECT index 57b3ef28..cb690308 100644 --- a/PROJECT +++ b/PROJECT @@ -1,3 +1,7 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: codeflare.dev layout: - go.kubebuilder.io/v3 @@ -6,4 +10,10 @@ plugins: scorecard.sdk.operatorframework.io/v2: {} projectName: codeflare-operator repo: github.com/project-codeflare/codeflare-operator +resources: +- controller: true + domain: codeflare.dev + group: ray + kind: RayCluster + version: v1 version: "3" diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index a74a16f1..12d4bb24 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -7,6 +7,8 @@ resources: - admin_role.yaml - editor_role.yaml - service_account.yaml +- mcad_manager_role.yaml +- mcad_manager_role_binding.yaml - role.yaml - role_binding.yaml - instascale_role.yaml diff --git a/config/rbac/mcad_manager_role.yaml b/config/rbac/mcad_manager_role.yaml new file mode 100644 index 00000000..b414b8e2 --- /dev/null +++ b/config/rbac/mcad_manager_role.yaml @@ -0,0 +1,223 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manual-manager-role +rules: +- apiGroups: + - '*' + resources: + - deployments + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments + - replicasets + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - config.openshift.io + resources: + - clusterversions + verbs: + - get + - list +- apiGroups: + - coordination.k8s.io + resources: + - kube-scheduler + - leases + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - bindings + - pods/binding + verbs: + - create +- apiGroups: + - "" + resources: + - configmaps + - nodes + - persistentvolumeclaims + - persistentvolumes + - secrets + - serviceaccounts + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + - kube-scheduler + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - kube-scheduler + verbs: + - get + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/status + verbs: + - patch + - update +- apiGroups: + - "" + resources: + - replicationcontrollers + verbs: + - get + - list + - watch +- apiGroups: + - events.k8s.io + resources: + - events + - kube-scheduler + verbs: + - create + - patch + - update +- apiGroups: + - machine.openshift.io + resources: + - '*' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - scheduling.sigs.k8s.io + resources: + - podgroups + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - storage.k8s.io + resources: + - csidrivers + - csinodes + - csistoragecapacities + verbs: + - get + - list + - watch +- apiGroups: + - workload.codeflare.dev + resources: + - appwrappers + - appwrappers/finalizers + - appwrappers/status + - schedulingspecs + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch +- apiGroups: + - quota.codeflare.dev + resources: + - quotasubtrees + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch diff --git a/config/rbac/mcad_manager_role_binding.yaml b/config/rbac/mcad_manager_role_binding.yaml new file mode 100644 index 00000000..af6c74ae --- /dev/null +++ b/config/rbac/mcad_manager_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manual-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manual-manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 07954b37..c7ef78b4 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,219 +5,62 @@ metadata: creationTimestamp: null name: manager-role rules: -- apiGroups: - - '*' - resources: - - deployments - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - deployments - - replicasets - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - config.openshift.io - resources: - - clusterversions - verbs: - - get - - list -- apiGroups: - - coordination.k8s.io - resources: - - kube-scheduler - - leases - verbs: - - create - - get - - update - apiGroups: - "" resources: - - bindings - - pods/binding - verbs: - - create -- apiGroups: - - "" - resources: - - configmaps - - nodes - - persistentvolumeclaims - - persistentvolumes - secrets - - serviceaccounts - - services verbs: - create - delete - get - - list - patch - - update - - watch - apiGroups: - "" resources: - - endpoints - - kube-scheduler - verbs: - - create - - get - - update -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update -- apiGroups: - - "" - resources: - - kube-scheduler - verbs: - - get - - update -- apiGroups: - - "" - resources: - - pods + - serviceaccounts verbs: - - create - delete - - deletecollection - get - - list - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/status - verbs: - - patch - - update - apiGroups: - "" resources: - - replicationcontrollers - verbs: - - get - - list - - watch -- apiGroups: - - events.k8s.io - resources: - - events - - kube-scheduler - verbs: - - create - - patch - - update -- apiGroups: - - machine.openshift.io - resources: - - '*' + - services verbs: - - create - delete - get - - list - patch - - update - - watch - apiGroups: - - scheduling.sigs.k8s.io + - ray.io resources: - - podgroups + - rayclusters verbs: - create - delete - - deletecollection - get - list - patch - update - watch - apiGroups: - - storage.k8s.io + - ray.io resources: - - csidrivers - - csinodes - - csistoragecapacities + - rayclusters/finalizers verbs: - - get - - list - - watch + - update - apiGroups: - - workload.codeflare.dev + - ray.io resources: - - appwrappers - - appwrappers/finalizers - - appwrappers/status - - schedulingspecs + - rayclusters/status verbs: - - create - - delete - - deletecollection - get - - list - patch - update - - watch - apiGroups: - - quota.codeflare.dev + - route.openshift.io resources: - - quotasubtrees + - routes verbs: - - create - delete - - deletecollection - get - - list - patch - - update - - watch diff --git a/controllers/raycluster_controller.go b/controllers/raycluster_controller.go new file mode 100644 index 00000000..00d7d559 --- /dev/null +++ b/controllers/raycluster_controller.go @@ -0,0 +1,338 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "crypto/rand" + "crypto/sha1" + "encoding/base64" + "strconv" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + coreapply "k8s.io/client-go/applyconfigurations/core/v1" + v1 "k8s.io/client-go/applyconfigurations/meta/v1" + rbacapply "k8s.io/client-go/applyconfigurations/rbac/v1" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + routev1 "github.com/openshift/api/route/v1" + routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" + routev1client "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1" +) + +// RayClusterReconciler reconciles a RayCluster object +type RayClusterReconciler struct { + client.Client + kubeClient *kubernetes.Clientset + routeClient *routev1client.RouteV1Client + Scheme *runtime.Scheme + CookieSalt string +} + +const ( + requeueTime = 10 + controllerName = "codeflare-raycluster-controller" + oauthAnnotation = "codeflare.dev/oauth=true" + CodeflareOAuthFinalizer = "codeflare.dev/oauth-finalizer" + OAuthServicePort = 443 + OAuthServicePortName = "oauth-proxy" + strTrue = "true" + strFalse = "false" + logRequeueing = "requeueing" +) + +var ( + deletePolicy = metav1.DeletePropagationForeground + deleteOptions = client.DeleteOptions{PropagationPolicy: &deletePolicy} +) + +//+kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizers,verbs=update +//+kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=patch;delete;get +//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get +//+kubebuilder:rbac:groups=core,resources=services,verbs=patch;delete;get +//+kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=patch;delete;get + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the RayCluster object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.15.3/pkg/reconcile + +func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := ctrl.LoggerFrom(ctx) + + var cluster rayv1.RayCluster + + if err := r.Get(ctx, req.NamespacedName, &cluster); err != nil { + if !errors.IsNotFound(err) { + logger.Error(err, "Error getting RayCluster resource") + } + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if cluster.ObjectMeta.DeletionTimestamp.IsZero() { + if !controllerutil.ContainsFinalizer(&cluster, CodeflareOAuthFinalizer) { + logger.Info("Add a finalizer", "finalizer", CodeflareOAuthFinalizer) + controllerutil.AddFinalizer(&cluster, CodeflareOAuthFinalizer) + if err := r.Update(ctx, &cluster); err != nil { + // this log is info level since errors are not fatal and are expected + logger.Info("WARN: Failed to update RayCluster with finalizer", "error", err.Error(), logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, err + } + } + } else if controllerutil.ContainsFinalizer(&cluster, CodeflareOAuthFinalizer) { + err := r.deleteIfNotExist( + ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: crbNameFromCluster(&cluster)}, &rbacv1.ClusterRoleBinding{}, + ) + if err != nil { + logger.Error(err, "Failed to remove OAuth ClusterRoleBinding.", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, err + } + controllerutil.RemoveFinalizer(&cluster, CodeflareOAuthFinalizer) + if err := r.Update(ctx, &cluster); err != nil { + logger.Error(err, "Failed to remove finalizer from RayCluster", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, err + } + logger.Info("Successfully removed finalizer.", logRequeueing, strFalse) + return ctrl.Result{}, nil + } + + val, ok := cluster.ObjectMeta.Annotations["codeflare.dev/oauth"] + boolVal, err := strconv.ParseBool(val) + if err != nil { + logger.Error(err, "Could not convert codeflare.dev/oauth value to bool", "codeflare.dev/oauth", val) + } + if !ok || err != nil || !boolVal { + logger.Info("Removing all OAuth Objects") + err := r.deleteIfNotExist( + ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: oauthSecretNameFromCluster(&cluster)}, &corev1.Secret{}, + ) + if err != nil { + logger.Error(err, "Error deleting OAuth Secret, retrying", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, nil + } + err = r.deleteIfNotExist( + ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: oauthServiceNameFromCluster(&cluster)}, &corev1.Service{}, + ) + if err != nil { + logger.Error(err, "Error deleting OAuth Service, retrying", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, nil + } + err = r.deleteIfNotExist( + ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: oauthServiceAccountNameFromCluster(&cluster)}, &corev1.ServiceAccount{}, + ) + if err != nil { + logger.Error(err, "Error deleting OAuth ServiceAccount, retrying", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, nil + } + err = r.deleteIfNotExist( + ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: crbNameFromCluster(&cluster)}, &rbacv1.ClusterRoleBinding{}, + ) + if err != nil { + logger.Error(err, "Error deleting OAuth CRB, retrying", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, nil + } + err = r.deleteIfNotExist( + ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: routeNameFromCluster(&cluster)}, &routev1.Route{}, + ) + if err != nil { + logger.Error(err, "Error deleting OAuth Route, retrying", logRequeueing, strTrue) + return ctrl.Result{RequeueAfter: requeueTime}, nil + } + return ctrl.Result{}, nil + } + + _, err = r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth Route") + } + + _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(&cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to create OAuth Secret") + } + + _, err = r.kubeClient.CoreV1().Services(cluster.Namespace).Apply(ctx, desiredOAuthService(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth Service") + } + + _, err = r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Apply(ctx, desiredServiceAccount(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth ServiceAccount") + } + + _, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth ClusterRoleBinding") + } + + return ctrl.Result{}, nil +} + +func crbNameFromCluster(cluster *rayv1.RayCluster) string { + return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz} +} + +func (r *RayClusterReconciler) deleteIfNotExist(ctx context.Context, namespacedName types.NamespacedName, obj client.Object) error { + err := r.Client.Get(ctx, namespacedName, obj) + if err != nil && errors.IsNotFound(err) { + return nil + } else if err != nil { + return err + } + return r.Client.Delete(ctx, obj, &deleteOptions) +} + +func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacapply.ClusterRoleBindingApplyConfiguration { + return rbacapply.ClusterRoleBinding( + crbNameFromCluster(cluster)). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithSubjects( + rbacapply.Subject(). + WithKind("ServiceAccount"). + WithName(oauthServiceAccountNameFromCluster(cluster)). + WithNamespace(cluster.Namespace), + ). + WithRoleRef( + rbacapply.RoleRef(). + WithAPIGroup("rbac.authorization.k8s.io"). + WithKind("ClusterRole"). + WithName("system:auth-delegator"), + ). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} + +func oauthServiceAccountNameFromCluster(cluster *rayv1.RayCluster) string { + return cluster.Name + "-oauth-proxy" +} + +func desiredServiceAccount(cluster *rayv1.RayCluster) *coreapply.ServiceAccountApplyConfiguration { + return coreapply.ServiceAccount(oauthServiceAccountNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithAnnotations(map[string]string{ + "serviceaccounts.openshift.io/oauth-redirectreference.first": "" + + `{"kind":"OAuthRedirectReference","apiVersion":"v1",` + + `"reference":{"kind":"Route","name":"` + routeNameFromCluster(cluster) + `"}}`, + }). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} + +func routeNameFromCluster(cluster *rayv1.RayCluster) string { + return "ray-dashboard-" + cluster.Name +} + +func desiredClusterRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration { + return routeapply.Route(routeNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithSpec(routeapply.RouteSpec(). + WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(oauthServiceNameFromCluster(cluster))). + WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString((OAuthServicePortName)))). + WithTLS(routeapply.TLSConfig(). + WithInsecureEdgeTerminationPolicy(routev1.InsecureEdgeTerminationPolicyRedirect). + WithTermination(routev1.TLSTerminationReencrypt), + ), + ). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} + +func oauthServiceNameFromCluster(cluster *rayv1.RayCluster) string { + return cluster.Name + "-oauth" +} + +func oauthServiceTLSSecretName(cluster *rayv1.RayCluster) string { + return cluster.Name + "-proxy-tls-secret" +} + +func desiredOAuthService(cluster *rayv1.RayCluster) *coreapply.ServiceApplyConfiguration { + return coreapply.Service(oauthServiceNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithAnnotations(map[string]string{"service.beta.openshift.io/serving-cert-secret-name": oauthServiceTLSSecretName(cluster)}). + WithSpec( + coreapply.ServiceSpec(). + WithPorts( + coreapply.ServicePort(). + WithName(OAuthServicePortName). + WithPort(OAuthServicePort). + WithTargetPort(intstr.FromString(OAuthServicePortName)). + WithProtocol(corev1.ProtocolTCP), + ). + WithSelector(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"}), + ). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} + +func oauthSecretNameFromCluster(cluster *rayv1.RayCluster) string { + return cluster.Name + "-oauth-config" +} + +// desiredOAuthSecret defines the desired OAuth secret object +func desiredOAuthSecret(cluster *rayv1.RayCluster, r *RayClusterReconciler) *coreapply.SecretApplyConfiguration { + // Generate the cookie secret for the OAuth proxy + hasher := sha1.New() // REVIEW is SHA1 okay here? + hasher.Write([]byte(cluster.Name + r.CookieSalt)) + cookieSecret := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) + + return coreapply.Secret(oauthSecretNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithStringData(map[string]string{"cookie_secret": cookieSecret}). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) + // Create a Kubernetes secret to store the cookie secret +} + +// SetupWithManager sets up the controller with the Manager. +func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.kubeClient = kubernetes.NewForConfigOrDie(mgr.GetConfig()) + r.routeClient = routev1client.NewForConfigOrDie(mgr.GetConfig()) + b := make([]byte, 16) + _, err := rand.Read(b) + if err != nil { + return err + } + r.CookieSalt = string(b) + return ctrl.NewControllerManagedBy(mgr). + Named(controllerName). + For(&rayv1.RayCluster{}). + Complete(r) +} diff --git a/controllers/raycluster_controller_test.go b/controllers/raycluster_controller_test.go new file mode 100644 index 00000000..0facb0cb --- /dev/null +++ b/controllers/raycluster_controller_test.go @@ -0,0 +1,222 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "math/rand" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + routev1 "github.com/openshift/api/route/v1" +) + +func stringInList(l []string, s string) bool { + for _, i := range l { + if i == s { + return true + } + } + return false +} + +var letters = []rune("abcdefghijklmnopqrstuvwxyz") +var r = rand.New(rand.NewSource(time.Now().UnixNano())) + +func randSeq(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letters[r.Intn(len(letters))] + } + return string(b) +} + +var _ = Describe("RayCluster controller", func() { + Context("RayCluster controller test", func() { + var rayClusterName = "test-raycluster" + var typeNamespaceName types.NamespacedName + ctx := context.Background() + BeforeEach(func() { + By("Generate random number so each run is creating unique") + rString := randSeq(10) + rayClusterName = rayClusterName + "-" + rString + typeNamespaceName = types.NamespacedName{Name: rayClusterName, Namespace: rayClusterName} + By("Creating a namespace for running the tests.") + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: rayClusterName, + }, + } + var err error + err = k8sClient.Create(ctx, namespace) + Expect(err).To(Not(HaveOccurred())) + + By("creating a basic instance of the RayCluster CR") + raycluster := &rayv1.RayCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: rayClusterName, + Namespace: rayClusterName, + }, + Spec: rayv1.RayClusterSpec{ + HeadGroupSpec: rayv1.HeadGroupSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + corev1.Container{}, + }, + }, + }, + RayStartParams: map[string]string{}, + }, + }, + } + err = k8sClient.Get(ctx, typeNamespaceName, &rayv1.RayCluster{}) + Expect(errors.IsNotFound(err)).To(Equal(true)) + err = k8sClient.Create(ctx, raycluster) + Expect(err).To(Not(HaveOccurred())) + }) + + AfterEach(func() { + By("removing the instance of the RayCluster used") + // err := clientSet.CoreV1().Namespaces().Delete(ctx, RayClusterName, metav1.DeleteOptions{}) + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + if err != nil { + Expect(errors.IsNotFound(err)).To(Equal(true)) + } else { + Expect(err).To(Not(HaveOccurred())) + _ = k8sClient.Delete(ctx, &foundRayCluster) + } + Eventually(func() bool { + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + return errors.IsNotFound(err) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + }) + + It("should have oauth finalizer set", func() { + foundRayCluster := rayv1.RayCluster{} + Eventually(func() bool { + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + return stringInList(foundRayCluster.Finalizers, CodeflareOAuthFinalizer) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + }) + + Context("Cluster has OAuth annotation", func() { + BeforeEach(func() { + By("adding OAuth annotation to RayCluster") + Eventually(func() error { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + if foundRayCluster.Annotations == nil { + foundRayCluster.Annotations = map[string]string{"codeflare.dev/oauth": "true"} + } else { + foundRayCluster.Annotations["codeflare.dev/oauth"] = "'true'" + } + return k8sClient.Update(ctx, &foundRayCluster) + }, SpecTimeout(time.Second*10)).Should(Not(HaveOccurred())) + By("waiting for dependent resources to be created") + Eventually(func() error { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + if err != nil { + return err + } + err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) + if err != nil { + return err + } + err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) + if err != nil { + return err + } + err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) + if err != nil { + return err + } + err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) + if err != nil { + return err + } + err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) + if err != nil { + return err + } + return nil + }, SpecTimeout(time.Second*10)).Should(Not(HaveOccurred())) + }) + + It("should set owner references for all resources", func() { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).ToNot(HaveOccurred()) + err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) + Expect(err).To(Not(HaveOccurred())) + }) + + It("should delete OAuth resources when annotation is removed", func() { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + delete(foundRayCluster.Annotations, "codeflare.dev/oauth") + err = k8sClient.Update(ctx, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{})) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{})) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{})) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{})) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + }) + + It("should remove CRB when the RayCluster is deleted", func() { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Delete(ctx, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{})) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) + }) + }) + }) +}) diff --git a/controllers/suite_test.go b/controllers/suite_test.go new file mode 100644 index 00000000..0110179f --- /dev/null +++ b/controllers/suite_test.go @@ -0,0 +1,135 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "io" + "net/http" + "os" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + routev1 "github.com/openshift/api/route/v1" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +const ( + RayClusterCRDFileDownload = "https://raw.githubusercontent.com/ray-project/kuberay/master/ray-operator/config/crd/bases/ray.io_rayclusters.yaml" + RouteCRDFileDownload = "https://raw.githubusercontent.com/openshift/api/master/route/v1/route.crd.yaml" +) + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + var err error + var fRoute, fRaycluster *os.File + + By("Creating and downloading necessary crds") + err = os.Mkdir("./test-crds", os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + fRoute, err = os.Create("./test-crds/route.yaml") + Expect(err).ToNot(HaveOccurred()) + defer fRoute.Close() + resp, err := http.Get(RouteCRDFileDownload) + Expect(err).ToNot(HaveOccurred()) + _, err = io.Copy(fRoute, resp.Body) + Expect(err).ToNot(HaveOccurred()) + fRaycluster, err = os.Create("./test-crds/raycluster.yaml") + Expect(err).ToNot(HaveOccurred()) + defer fRaycluster.Close() + resp, err = http.Get(RayClusterCRDFileDownload) + Expect(err).ToNot(HaveOccurred()) + _, err = io.Copy(fRaycluster, resp.Body) + Expect(err).ToNot(HaveOccurred()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "config", "crd"), + filepath.Join(".", "test-crds"), + }, + ErrorIfCRDPathMissing: true, + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + clientSet, err := kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + err = rayv1.AddToScheme(scheme.Scheme) + Expect(err).To(Not(HaveOccurred())) + err = routev1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + }) + Expect(err).NotTo(HaveOccurred()) + err = (&RayClusterReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + kubeClient: clientSet, + CookieSalt: "foo", + }).SetupWithManager(k8sManager) + Expect(err).NotTo(HaveOccurred()) + go func() { + defer GinkgoRecover() + err = k8sManager.Start(context.Background()) + Expect(err).ToNot(HaveOccurred(), "failed to run manager") + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := os.RemoveAll("./test-crds") + Expect(err).NotTo(HaveOccurred()) + err = testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/go.mod b/go.mod index 7f846b2b..93fa2c9f 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,11 @@ module github.com/project-codeflare/codeflare-operator go 1.20 require ( + github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 - github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 + github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c + github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a github.com/project-codeflare/instascale v0.4.0 github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0 github.com/ray-project/kuberay/ray-operator v1.0.0 @@ -46,6 +48,7 @@ require ( github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/glog v1.1.2 // indirect @@ -55,6 +58,7 @@ require ( github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect @@ -72,7 +76,6 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect - github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect @@ -102,6 +105,7 @@ require ( golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect diff --git a/go.sum b/go.sum index 60345cd5..7d5f4c95 100644 --- a/go.sum +++ b/go.sum @@ -142,6 +142,7 @@ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -216,6 +217,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -366,6 +368,7 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= @@ -384,8 +387,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303 h1:30LG8751WElZmWA3mVS8l23l2oZnUCqbDkLCyy0U/p0= -github.com/project-codeflare/codeflare-common v0.0.0-20240201153809-2e7292120303/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= +github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a h1:Yk9J5qXjp+yfSRCzS0EElrhpTgfYJ+S+W/z84cmlmX4= +github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo= github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo= github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0 h1:IkTmd/W/zxcsC5s4EbnW74PFpkQVEiTc/8rWWwFw0Ok= @@ -450,6 +453,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -569,6 +573,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -764,6 +769,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index 5e2bb6e3..58aa0289 100644 --- a/main.go +++ b/main.go @@ -30,13 +30,16 @@ import ( quotasubtreev1alpha1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/quotaplugins/quotasubtree/v1alpha1" mcadconfig "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config" mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" "go.uber.org/zap/zapcore" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -51,7 +54,9 @@ import ( configv1 "github.com/openshift/api/config/v1" machinev1beta1 "github.com/openshift/api/machine/v1beta1" + routev1 "github.com/openshift/api/route/v1" + cfoControllers "github.com/project-codeflare/codeflare-operator/controllers" "github.com/project-codeflare/codeflare-operator/pkg/config" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -75,6 +80,10 @@ func init() { // InstaScale utilruntime.Must(configv1.Install(scheme)) utilruntime.Must(machinev1beta1.Install(scheme)) + // Ray + utilruntime.Must(rayv1.AddToScheme(scheme)) + // OpenShift Route + utilruntime.Must(routev1.Install(scheme)) } func main() { @@ -171,6 +180,13 @@ func main() { exitOnError(instaScaleController.SetupWithManager(context.Background(), mgr), "Error setting up InstaScale controller") } + if v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")); v { + rayClusterController := cfoControllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} + exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") + } else if err != nil { + exitOnError(err, "Could not determine if RayCluster CR present on cluster.") + } + exitOnError(mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping), "unable to set up health check") exitOnError(mgr.AddReadyzCheck(cfg.Health.ReadinessEndpointName, healthz.Ping), "unable to set up ready check") @@ -221,6 +237,23 @@ func createConfigMap(ctx context.Context, client kubernetes.Interface, ns, name return err } +func HasAPIResourceForGVK(dc discovery.DiscoveryInterface, gvk schema.GroupVersionKind) (bool, error) { + gv, kind := gvk.ToAPIVersionAndKind() + if resources, err := dc.ServerResourcesForGroupVersion(gv); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } else { + for _, res := range resources.APIResources { + if res.Kind == kind { + return true, nil + } + } + } + return false, nil +} + func namespaceOrDie() string { // This way assumes you've set the NAMESPACE environment variable either manually, when running // the operator standalone, or using the downward API, when running the operator in-cluster. From 4f0c9c1df24fe712c43d718f1340dd1c5f95825f Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 2 Apr 2024 10:09:46 -0400 Subject: [PATCH 306/377] make oauth part of config and remove annotation check Signed-off-by: Kevin --- controllers/raycluster_controller.go | 48 +------- controllers/raycluster_controller_test.go | 132 ++++++++-------------- main.go | 4 +- pkg/config/config.go | 2 + 4 files changed, 54 insertions(+), 132 deletions(-) diff --git a/controllers/raycluster_controller.go b/controllers/raycluster_controller.go index 00d7d559..54ebf07a 100644 --- a/controllers/raycluster_controller.go +++ b/controllers/raycluster_controller.go @@ -21,7 +21,6 @@ import ( "crypto/rand" "crypto/sha1" "encoding/base64" - "strconv" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -128,52 +127,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - val, ok := cluster.ObjectMeta.Annotations["codeflare.dev/oauth"] - boolVal, err := strconv.ParseBool(val) - if err != nil { - logger.Error(err, "Could not convert codeflare.dev/oauth value to bool", "codeflare.dev/oauth", val) - } - if !ok || err != nil || !boolVal { - logger.Info("Removing all OAuth Objects") - err := r.deleteIfNotExist( - ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: oauthSecretNameFromCluster(&cluster)}, &corev1.Secret{}, - ) - if err != nil { - logger.Error(err, "Error deleting OAuth Secret, retrying", logRequeueing, strTrue) - return ctrl.Result{RequeueAfter: requeueTime}, nil - } - err = r.deleteIfNotExist( - ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: oauthServiceNameFromCluster(&cluster)}, &corev1.Service{}, - ) - if err != nil { - logger.Error(err, "Error deleting OAuth Service, retrying", logRequeueing, strTrue) - return ctrl.Result{RequeueAfter: requeueTime}, nil - } - err = r.deleteIfNotExist( - ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: oauthServiceAccountNameFromCluster(&cluster)}, &corev1.ServiceAccount{}, - ) - if err != nil { - logger.Error(err, "Error deleting OAuth ServiceAccount, retrying", logRequeueing, strTrue) - return ctrl.Result{RequeueAfter: requeueTime}, nil - } - err = r.deleteIfNotExist( - ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: crbNameFromCluster(&cluster)}, &rbacv1.ClusterRoleBinding{}, - ) - if err != nil { - logger.Error(err, "Error deleting OAuth CRB, retrying", logRequeueing, strTrue) - return ctrl.Result{RequeueAfter: requeueTime}, nil - } - err = r.deleteIfNotExist( - ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: routeNameFromCluster(&cluster)}, &routev1.Route{}, - ) - if err != nil { - logger.Error(err, "Error deleting OAuth Route, retrying", logRequeueing, strTrue) - return ctrl.Result{RequeueAfter: requeueTime}, nil - } - return ctrl.Result{}, nil - } - - _, err = r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update OAuth Route") } diff --git a/controllers/raycluster_controller_test.go b/controllers/raycluster_controller_test.go index 0facb0cb..fbbaaa46 100644 --- a/controllers/raycluster_controller_test.go +++ b/controllers/raycluster_controller_test.go @@ -125,98 +125,62 @@ var _ = Describe("RayCluster controller", func() { }, SpecTimeout(time.Second*10)).Should(Equal(true)) }) - Context("Cluster has OAuth annotation", func() { - BeforeEach(func() { - By("adding OAuth annotation to RayCluster") - Eventually(func() error { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - Expect(err).To(Not(HaveOccurred())) - if foundRayCluster.Annotations == nil { - foundRayCluster.Annotations = map[string]string{"codeflare.dev/oauth": "true"} - } else { - foundRayCluster.Annotations["codeflare.dev/oauth"] = "'true'" - } - return k8sClient.Update(ctx, &foundRayCluster) - }, SpecTimeout(time.Second*10)).Should(Not(HaveOccurred())) - By("waiting for dependent resources to be created") - Eventually(func() error { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) - if err != nil { - return err - } - return nil - }, SpecTimeout(time.Second*10)).Should(Not(HaveOccurred())) - }) - - It("should set owner references for all resources", func() { + It("should create all oauth resources", func() { + Eventually(func() error { foundRayCluster := rayv1.RayCluster{} err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - Expect(err).ToNot(HaveOccurred()) + if err != nil { + return err + } err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) - Expect(err).To(Not(HaveOccurred())) + if err != nil { + return err + } err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) - Expect(err).To(Not(HaveOccurred())) + if err != nil { + return err + } err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) - Expect(err).To(Not(HaveOccurred())) + if err != nil { + return err + } err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) - Expect(err).To(Not(HaveOccurred())) + if err != nil { + return err + } err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) - Expect(err).To(Not(HaveOccurred())) - }) + if err != nil { + return err + } + return nil + }, SpecTimeout(time.Second*10)).Should(Not(HaveOccurred())) + }) - It("should delete OAuth resources when annotation is removed", func() { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - Expect(err).To(Not(HaveOccurred())) - delete(foundRayCluster.Annotations, "codeflare.dev/oauth") - err = k8sClient.Update(ctx, &foundRayCluster) - Expect(err).To(Not(HaveOccurred())) - Eventually(func() bool { - return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{})) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) - Eventually(func() bool { - return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{})) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) - Eventually(func() bool { - return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{})) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) - Eventually(func() bool { - return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{})) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) - }) - - It("should remove CRB when the RayCluster is deleted", func() { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - Expect(err).To(Not(HaveOccurred())) - err = k8sClient.Delete(ctx, &foundRayCluster) - Expect(err).To(Not(HaveOccurred())) - Eventually(func() bool { - return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{})) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) - }) + It("should set owner references for all resources", func() { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).ToNot(HaveOccurred()) + err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) + Expect(err).To(Not(HaveOccurred())) + }) + + It("should remove CRB when the RayCluster is deleted", func() { + foundRayCluster := rayv1.RayCluster{} + err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + err = k8sClient.Delete(ctx, &foundRayCluster) + Expect(err).To(Not(HaveOccurred())) + Eventually(func() bool { + return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{})) + }, SpecTimeout(time.Second*10)).Should(Equal(true)) }) }) }) diff --git a/main.go b/main.go index 58aa0289..951dbf98 100644 --- a/main.go +++ b/main.go @@ -134,6 +134,7 @@ func main() { MaxScaleoutAllowed: 5, }, }, + RayClusterOAuth: pointer.Bool(true), } kubeConfig, err := ctrl.GetConfig() @@ -180,7 +181,8 @@ func main() { exitOnError(instaScaleController.SetupWithManager(context.Background(), mgr), "Error setting up InstaScale controller") } - if v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")); v { + v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) + if v && *cfg.RayClusterOAuth { rayClusterController := cfoControllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index 295e3e00..79a944bf 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -36,6 +36,8 @@ type CodeFlareOperatorConfiguration struct { // The InstaScale controller configuration InstaScale *InstaScaleConfiguration `json:"instascale,omitempty"` + + RayClusterOAuth *bool `json:"rayClusterOAuth,omitempty"` } type InstaScaleConfiguration struct { From 9ea041fc68aad6ba5eb9d9509d783b0307905ca3 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 2 Apr 2024 12:26:51 -0400 Subject: [PATCH 307/377] address comments from PR review Signed-off-by: Kevin --- controllers/raycluster_controller.go | 36 +++++++++++----------------- main.go | 6 +++-- pkg/config/config.go | 6 ++++- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/controllers/raycluster_controller.go b/controllers/raycluster_controller.go index 54ebf07a..8d7299b5 100644 --- a/controllers/raycluster_controller.go +++ b/controllers/raycluster_controller.go @@ -29,7 +29,6 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" coreapply "k8s.io/client-go/applyconfigurations/core/v1" v1 "k8s.io/client-go/applyconfigurations/meta/v1" @@ -56,12 +55,9 @@ type RayClusterReconciler struct { const ( requeueTime = 10 controllerName = "codeflare-raycluster-controller" - oauthAnnotation = "codeflare.dev/oauth=true" - CodeflareOAuthFinalizer = "codeflare.dev/oauth-finalizer" + CodeflareOAuthFinalizer = "openshift.ai/oauth-finalizer" OAuthServicePort = 443 OAuthServicePortName = "oauth-proxy" - strTrue = "true" - strFalse = "false" logRequeueing = "requeueing" ) @@ -106,24 +102,30 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) controllerutil.AddFinalizer(&cluster, CodeflareOAuthFinalizer) if err := r.Update(ctx, &cluster); err != nil { // this log is info level since errors are not fatal and are expected - logger.Info("WARN: Failed to update RayCluster with finalizer", "error", err.Error(), logRequeueing, strTrue) + logger.Info("WARN: Failed to update RayCluster with finalizer", "error", err.Error(), logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } } } else if controllerutil.ContainsFinalizer(&cluster, CodeflareOAuthFinalizer) { - err := r.deleteIfNotExist( - ctx, types.NamespacedName{Namespace: cluster.Namespace, Name: crbNameFromCluster(&cluster)}, &rbacv1.ClusterRoleBinding{}, - ) + err := client.IgnoreNotFound(r.Client.Delete( + ctx, + &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: crbNameFromCluster(&cluster), + }, + }, + &deleteOptions, + )) if err != nil { - logger.Error(err, "Failed to remove OAuth ClusterRoleBinding.", logRequeueing, strTrue) + logger.Error(err, "Failed to remove OAuth ClusterRoleBinding.", logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } controllerutil.RemoveFinalizer(&cluster, CodeflareOAuthFinalizer) if err := r.Update(ctx, &cluster); err != nil { - logger.Error(err, "Failed to remove finalizer from RayCluster", logRequeueing, strTrue) + logger.Error(err, "Failed to remove finalizer from RayCluster", logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } - logger.Info("Successfully removed finalizer.", logRequeueing, strFalse) + logger.Info("Successfully removed finalizer.", logRequeueing, false) return ctrl.Result{}, nil } @@ -159,16 +161,6 @@ func crbNameFromCluster(cluster *rayv1.RayCluster) string { return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz} } -func (r *RayClusterReconciler) deleteIfNotExist(ctx context.Context, namespacedName types.NamespacedName, obj client.Object) error { - err := r.Client.Get(ctx, namespacedName, obj) - if err != nil && errors.IsNotFound(err) { - return nil - } else if err != nil { - return err - } - return r.Client.Delete(ctx, obj, &deleteOptions) -} - func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacapply.ClusterRoleBindingApplyConfiguration { return rbacapply.ClusterRoleBinding( crbNameFromCluster(cluster)). diff --git a/main.go b/main.go index 951dbf98..99b293cd 100644 --- a/main.go +++ b/main.go @@ -134,7 +134,9 @@ func main() { MaxScaleoutAllowed: 5, }, }, - RayClusterOAuth: pointer.Bool(true), + KubeRay: &config.KubeRayConfiguration{ + RayDashboardOAuthEnabled: pointer.Bool(true), + }, } kubeConfig, err := ctrl.GetConfig() @@ -182,7 +184,7 @@ func main() { } v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) - if v && *cfg.RayClusterOAuth { + if v && *cfg.KubeRay.RayDashboardOAuthEnabled { rayClusterController := cfoControllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index 79a944bf..10b54767 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -37,7 +37,11 @@ type CodeFlareOperatorConfiguration struct { // The InstaScale controller configuration InstaScale *InstaScaleConfiguration `json:"instascale,omitempty"` - RayClusterOAuth *bool `json:"rayClusterOAuth,omitempty"` + KubeRay *KubeRayConfiguration `json:"kuberay,omitempty"` +} + +type KubeRayConfiguration struct { + RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"` } type InstaScaleConfiguration struct { From 5e5f5b30b9908befb0dcff0b58c1cd2d8e40895d Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 3 Apr 2024 14:45:58 +0200 Subject: [PATCH 308/377] Move controllers package into pkg --- Dockerfile | 1 - main.go | 4 ++-- {controllers => pkg/controllers}/raycluster_controller.go | 0 .../controllers}/raycluster_controller_test.go | 0 {controllers => pkg/controllers}/suite_test.go | 0 5 files changed, 2 insertions(+), 3 deletions(-) rename {controllers => pkg/controllers}/raycluster_controller.go (100%) rename {controllers => pkg/controllers}/raycluster_controller_test.go (100%) rename {controllers => pkg/controllers}/suite_test.go (100%) diff --git a/Dockerfile b/Dockerfile index 26f4c23d..f5c51c3f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,6 @@ RUN go mod download # Copy the Go sources COPY main.go main.go COPY pkg/ pkg/ -COPY controllers/ controllers/ # Build USER root diff --git a/main.go b/main.go index 99b293cd..04cd33ec 100644 --- a/main.go +++ b/main.go @@ -56,8 +56,8 @@ import ( machinev1beta1 "github.com/openshift/api/machine/v1beta1" routev1 "github.com/openshift/api/route/v1" - cfoControllers "github.com/project-codeflare/codeflare-operator/controllers" "github.com/project-codeflare/codeflare-operator/pkg/config" + "github.com/project-codeflare/codeflare-operator/pkg/controllers" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. // +kubebuilder:scaffold:imports @@ -185,7 +185,7 @@ func main() { v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) if v && *cfg.KubeRay.RayDashboardOAuthEnabled { - rayClusterController := cfoControllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} + rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { exitOnError(err, "Could not determine if RayCluster CR present on cluster.") diff --git a/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go similarity index 100% rename from controllers/raycluster_controller.go rename to pkg/controllers/raycluster_controller.go diff --git a/controllers/raycluster_controller_test.go b/pkg/controllers/raycluster_controller_test.go similarity index 100% rename from controllers/raycluster_controller_test.go rename to pkg/controllers/raycluster_controller_test.go diff --git a/controllers/suite_test.go b/pkg/controllers/suite_test.go similarity index 100% rename from controllers/suite_test.go rename to pkg/controllers/suite_test.go From 2f100afe80d753dfd13518cc74088df0fa2153b6 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 3 Apr 2024 14:49:39 +0200 Subject: [PATCH 309/377] Scope RayCluster finalizer into ray.openshift.ai --- pkg/controllers/raycluster_controller.go | 44 +++++++++---------- pkg/controllers/raycluster_controller_test.go | 2 +- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 8d7299b5..53999591 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -53,12 +53,12 @@ type RayClusterReconciler struct { } const ( - requeueTime = 10 - controllerName = "codeflare-raycluster-controller" - CodeflareOAuthFinalizer = "openshift.ai/oauth-finalizer" - OAuthServicePort = 443 - OAuthServicePortName = "oauth-proxy" - logRequeueing = "requeueing" + requeueTime = 10 + controllerName = "codeflare-raycluster-controller" + oAuthFinalizer = "ray.openshift.ai/oauth-finalizer" + oAuthServicePort = 443 + oAuthServicePortName = "oauth-proxy" + logRequeueing = "requeueing" ) var ( @@ -66,13 +66,13 @@ var ( deleteOptions = client.DeleteOptions{PropagationPolicy: &deletePolicy} ) -//+kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizers,verbs=update -//+kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=patch;delete;get -//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get -//+kubebuilder:rbac:groups=core,resources=services,verbs=patch;delete;get -//+kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=patch;delete;get +// +kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizers,verbs=update +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=patch;delete;get +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get +// +kubebuilder:rbac:groups=core,resources=services,verbs=patch;delete;get +// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=patch;delete;get // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -97,16 +97,16 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if cluster.ObjectMeta.DeletionTimestamp.IsZero() { - if !controllerutil.ContainsFinalizer(&cluster, CodeflareOAuthFinalizer) { - logger.Info("Add a finalizer", "finalizer", CodeflareOAuthFinalizer) - controllerutil.AddFinalizer(&cluster, CodeflareOAuthFinalizer) + if !controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { + logger.Info("Add a finalizer", "finalizer", oAuthFinalizer) + controllerutil.AddFinalizer(&cluster, oAuthFinalizer) if err := r.Update(ctx, &cluster); err != nil { // this log is info level since errors are not fatal and are expected logger.Info("WARN: Failed to update RayCluster with finalizer", "error", err.Error(), logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } } - } else if controllerutil.ContainsFinalizer(&cluster, CodeflareOAuthFinalizer) { + } else if controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { err := client.IgnoreNotFound(r.Client.Delete( ctx, &rbacv1.ClusterRoleBinding{ @@ -120,7 +120,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Error(err, "Failed to remove OAuth ClusterRoleBinding.", logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } - controllerutil.RemoveFinalizer(&cluster, CodeflareOAuthFinalizer) + controllerutil.RemoveFinalizer(&cluster, oAuthFinalizer) if err := r.Update(ctx, &cluster); err != nil { logger.Error(err, "Failed to remove finalizer from RayCluster", logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err @@ -208,7 +208,7 @@ func desiredClusterRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfig WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithSpec(routeapply.RouteSpec(). WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(oauthServiceNameFromCluster(cluster))). - WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString((OAuthServicePortName)))). + WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString((oAuthServicePortName)))). WithTLS(routeapply.TLSConfig(). WithInsecureEdgeTerminationPolicy(routev1.InsecureEdgeTerminationPolicyRedirect). WithTermination(routev1.TLSTerminationReencrypt), @@ -235,9 +235,9 @@ func desiredOAuthService(cluster *rayv1.RayCluster) *coreapply.ServiceApplyConfi coreapply.ServiceSpec(). WithPorts( coreapply.ServicePort(). - WithName(OAuthServicePortName). - WithPort(OAuthServicePort). - WithTargetPort(intstr.FromString(OAuthServicePortName)). + WithName(oAuthServicePortName). + WithPort(oAuthServicePort). + WithTargetPort(intstr.FromString(oAuthServicePortName)). WithProtocol(corev1.ProtocolTCP), ). WithSelector(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"}), diff --git a/pkg/controllers/raycluster_controller_test.go b/pkg/controllers/raycluster_controller_test.go index fbbaaa46..6592aa29 100644 --- a/pkg/controllers/raycluster_controller_test.go +++ b/pkg/controllers/raycluster_controller_test.go @@ -121,7 +121,7 @@ var _ = Describe("RayCluster controller", func() { Eventually(func() bool { err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) Expect(err).To(Not(HaveOccurred())) - return stringInList(foundRayCluster.Finalizers, CodeflareOAuthFinalizer) + return stringInList(foundRayCluster.Finalizers, oAuthFinalizer) }, SpecTimeout(time.Second*10)).Should(Equal(true)) }) From 22d75f6b8ab1f6ee36c0340b9c0a842f8c4048a3 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 3 Apr 2024 18:49:55 +0200 Subject: [PATCH 310/377] Add missing RBAC for oauth-proxy ClusterRoleBinding --- config/rbac/role.yaml | 8 ++++++++ pkg/controllers/raycluster_controller.go | 1 + 2 files changed, 9 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c7ef78b4..e5a2ca9f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -56,6 +56,14 @@ rules: - get - patch - update +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + verbs: + - delete + - get + - patch - apiGroups: - route.openshift.io resources: diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 53999591..9e45bf78 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -73,6 +73,7 @@ var ( // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get // +kubebuilder:rbac:groups=core,resources=services,verbs=patch;delete;get // +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=patch;delete;get +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=patch;delete;get // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. From a8e2a9315e545fa5a0cfd88a42adcd988e3a48ed Mon Sep 17 00:00:00 2001 From: Anish Asthana Date: Thu, 4 Apr 2024 16:03:29 -0400 Subject: [PATCH 311/377] Disable MCAD by default Signed-off-by: Anish Asthana --- main.go | 15 +++++++++------ pkg/config/config.go | 4 +++- test/e2e/setup.sh | 10 ++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 04cd33ec..42cc5a3e 100644 --- a/main.go +++ b/main.go @@ -127,7 +127,8 @@ func main() { }, LeaderElection: &configv1alpha1.LeaderElectionConfiguration{}, }, - MCAD: &mcadconfig.MCADConfiguration{}, + MCADEnabled: pointer.Bool(false), + MCAD: &mcadconfig.MCADConfiguration{}, InstaScale: &config.InstaScaleConfiguration{ Enabled: pointer.Bool(false), InstaScaleConfiguration: instascaleconfig.InstaScaleConfiguration{ @@ -167,12 +168,14 @@ func main() { }) exitOnError(err, "unable to start manager") - mcadQueueController := mcad.NewJobController(mgr.GetConfig(), cfg.MCAD, &mcadconfig.MCADConfigurationExtended{}) - if mcadQueueController == nil { - // FIXME: update NewJobController so it follows Go idiomatic error handling and return an error instead of a nil object - os.Exit(1) + if pointer.BoolDeref(cfg.MCADEnabled, false) { + mcadQueueController := mcad.NewJobController(mgr.GetConfig(), cfg.MCAD, &mcadconfig.MCADConfigurationExtended{}) + if mcadQueueController == nil { + // FIXME: update NewJobController so it follows Go idiomatic error handling and return an error instead of a nil object + os.Exit(1) + } + mcadQueueController.Run(ctx.Done()) } - mcadQueueController.Run(ctx.Done()) if pointer.BoolDeref(cfg.InstaScale.Enabled, false) { instaScaleController := &instascale.AppWrapperReconciler{ diff --git a/pkg/config/config.go b/pkg/config/config.go index 10b54767..36234a3c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -32,7 +32,9 @@ type CodeFlareOperatorConfiguration struct { ControllerManager `json:",inline"` // The MCAD controller configuration - MCAD *mcad.MCADConfiguration `json:"mcad,omitempty"` + // MCADEnabled defaults to false + MCADEnabled *bool `json:"mcadEnabled,omitempty"` + MCAD *mcad.MCADConfiguration `json:"mcad,omitempty"` // The InstaScale controller configuration InstaScale *InstaScaleConfiguration `json:"instascale,omitempty"` diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index d5c99cbf..1f8922aa 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -56,3 +56,13 @@ roleRef: kind: ClusterRole name: mcad-controller-rayclusters EOF + +cat < Date: Fri, 5 Apr 2024 09:32:19 +0200 Subject: [PATCH 312/377] Enable MCAD in e2e tests --- config/e2e/config.yaml | 8 ++++++++ config/e2e/kustomization.yaml | 1 + main.go | 9 +++++---- pkg/config/config.go | 13 ++++++++++--- test/e2e/setup.sh | 10 ---------- 5 files changed, 24 insertions(+), 17 deletions(-) create mode 100644 config/e2e/config.yaml diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml new file mode 100644 index 00000000..4b01a4b0 --- /dev/null +++ b/config/e2e/config.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: codeflare-operator-config +data: + config.yaml: | + mcad: + enabled: true diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml index e06123b2..772370da 100644 --- a/config/e2e/kustomization.yaml +++ b/config/e2e/kustomization.yaml @@ -1,6 +1,7 @@ namespace: openshift-operators bases: +- config.yaml - ../default patches: diff --git a/main.go b/main.go index 42cc5a3e..05a53f07 100644 --- a/main.go +++ b/main.go @@ -127,8 +127,9 @@ func main() { }, LeaderElection: &configv1alpha1.LeaderElectionConfiguration{}, }, - MCADEnabled: pointer.Bool(false), - MCAD: &mcadconfig.MCADConfiguration{}, + MCAD: &config.MCADConfiguration{ + Enabled: pointer.Bool(false), + }, InstaScale: &config.InstaScaleConfiguration{ Enabled: pointer.Bool(false), InstaScaleConfiguration: instascaleconfig.InstaScaleConfiguration{ @@ -168,8 +169,8 @@ func main() { }) exitOnError(err, "unable to start manager") - if pointer.BoolDeref(cfg.MCADEnabled, false) { - mcadQueueController := mcad.NewJobController(mgr.GetConfig(), cfg.MCAD, &mcadconfig.MCADConfigurationExtended{}) + if pointer.BoolDeref(cfg.MCAD.Enabled, false) { + mcadQueueController := mcad.NewJobController(mgr.GetConfig(), &cfg.MCAD.MCADConfiguration, &mcadconfig.MCADConfigurationExtended{}) if mcadQueueController == nil { // FIXME: update NewJobController so it follows Go idiomatic error handling and return an error instead of a nil object os.Exit(1) diff --git a/pkg/config/config.go b/pkg/config/config.go index 36234a3c..85a03385 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -32,9 +32,7 @@ type CodeFlareOperatorConfiguration struct { ControllerManager `json:",inline"` // The MCAD controller configuration - // MCADEnabled defaults to false - MCADEnabled *bool `json:"mcadEnabled,omitempty"` - MCAD *mcad.MCADConfiguration `json:"mcad,omitempty"` + MCAD *MCADConfiguration `json:"mcad,omitempty"` // The InstaScale controller configuration InstaScale *InstaScaleConfiguration `json:"instascale,omitempty"` @@ -46,6 +44,15 @@ type KubeRayConfiguration struct { RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"` } +type MCADConfiguration struct { + // enabled controls whether the MCAD controller is started. + // It defaults to false. + Enabled *bool `json:"enabled,omitempty"` + + // The InstaScale controller configuration + mcad.MCADConfiguration `json:",inline,omitempty"` +} + type InstaScaleConfiguration struct { // enabled controls whether the InstaScale controller is started. // It may default to true on platforms that InstaScale supports. diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index 1f8922aa..d5c99cbf 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -56,13 +56,3 @@ roleRef: kind: ClusterRole name: mcad-controller-rayclusters EOF - -cat < Date: Fri, 5 Apr 2024 10:03:18 +0200 Subject: [PATCH 313/377] Enable MCAD in OLM tests --- .github/workflows/olm_tests.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index aa8a3b17..b6932d89 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -92,6 +92,10 @@ jobs: echo Waiting for Deployment to be ready timeout 60 bash -c 'until [[ $(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}') ]]; do sleep 5 && echo "$(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}')"; done' kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} deployment/codeflare-operator-manager --for=condition=Available=true --timeout=60s + + echo Patch the CodeFlare operator ConfigMap + kubectl patch -n '${{ env.SUBSCRIPTION_NAMESPACE }}' cm codeflare-operator-config --type merge -p '{"data":{"config.yaml":"mcad:\n enabled: true"}}' + env: CATALOG_SOURCE_NAME: "codeflare-olm-test" CATALOG_SOURCE_NAMESPACE: "olm" From 465da2068475f437dba4b41736fda7dc3c62d25a Mon Sep 17 00:00:00 2001 From: Christian Zaccaria <73656840+ChristianZaccaria@users.noreply.github.com> Date: Fri, 5 Apr 2024 16:31:21 +0100 Subject: [PATCH 314/377] Refactor creation logic of ingress/routes into RayCluster Controller (#493) * WIP - Creation/Deletion of ingress and routes in RayCluster Controller * Always run RayCluster Controller * Add support file for RC Controller * Disable OAuth for e2e tests * Add OAuth config to reconciler struct --- .github/workflows/e2e_tests.yaml | 1 + main.go | 4 +- pkg/controllers/raycluster_controller.go | 102 +++++++++---- pkg/controllers/support.go | 181 +++++++++++++++++++++++ 4 files changed, 258 insertions(+), 30 deletions(-) create mode 100644 pkg/controllers/support.go diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 0fc1ec88..2e8fdbc1 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -60,6 +60,7 @@ jobs: run: | echo Deploying CodeFlare operator IMG="${REGISTRY_ADDRESS}"/codeflare-operator + sed -i 's/RayDashboardOAuthEnabled: pointer.Bool(true)/RayDashboardOAuthEnabled: pointer.Bool(false)/' main.go make image-push -e IMG="${IMG}" make deploy -e IMG="${IMG}" -e ENV="e2e" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager diff --git a/main.go b/main.go index 05a53f07..658b55e9 100644 --- a/main.go +++ b/main.go @@ -188,8 +188,8 @@ func main() { } v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) - if v && *cfg.KubeRay.RayDashboardOAuthEnabled { - rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()} + if v { + rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg} exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { exitOnError(err, "Could not determine if RayCluster CR present on cluster.") diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 9e45bf78..23104c2a 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -41,6 +41,8 @@ import ( routev1 "github.com/openshift/api/route/v1" routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" routev1client "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1" + + "github.com/project-codeflare/codeflare-operator/pkg/config" ) // RayClusterReconciler reconciles a RayCluster object @@ -50,15 +52,17 @@ type RayClusterReconciler struct { routeClient *routev1client.RouteV1Client Scheme *runtime.Scheme CookieSalt string + Config *config.CodeFlareOperatorConfiguration } const ( - requeueTime = 10 - controllerName = "codeflare-raycluster-controller" - oAuthFinalizer = "ray.openshift.ai/oauth-finalizer" - oAuthServicePort = 443 - oAuthServicePortName = "oauth-proxy" - logRequeueing = "requeueing" + requeueTime = 10 + controllerName = "codeflare-raycluster-controller" + oAuthFinalizer = "ray.openshift.ai/oauth-finalizer" + oAuthServicePort = 443 + oAuthServicePortName = "oauth-proxy" + ingressServicePortName = "dashboard" + logRequeueing = "requeueing" ) var ( @@ -97,6 +101,10 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, client.IgnoreNotFound(err) } + isLocalInteractive := annotationBoolVal(ctx, &cluster, "sdk.codeflare.dev/local_interactive", false) + ingressDomain := "" // FIX - CFO will retrieve it. + isOpenShift, ingressHost := getClusterType(ctx, r.kubeClient, &cluster, ingressDomain) + if cluster.ObjectMeta.DeletionTimestamp.IsZero() { if !controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { logger.Info("Add a finalizer", "finalizer", oAuthFinalizer) @@ -130,29 +138,63 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to update OAuth Route") - } + if cluster.Status.State != "suspended" && r.isRayDashboardOAuthEnabled() && isOpenShift { + logger.Info("Creating OAuth Objects") + _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth Route") + return ctrl.Result{RequeueAfter: requeueTime}, err + } - _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(&cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to create OAuth Secret") - } + _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(&cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to create OAuth Secret") + return ctrl.Result{RequeueAfter: requeueTime}, err + } - _, err = r.kubeClient.CoreV1().Services(cluster.Namespace).Apply(ctx, desiredOAuthService(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to update OAuth Service") - } + _, err = r.kubeClient.CoreV1().Services(cluster.Namespace).Apply(ctx, desiredOAuthService(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth Service") + return ctrl.Result{RequeueAfter: requeueTime}, err + } - _, err = r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Apply(ctx, desiredServiceAccount(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to update OAuth ServiceAccount") - } + _, err = r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Apply(ctx, desiredServiceAccount(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth ServiceAccount") + return ctrl.Result{RequeueAfter: requeueTime}, err + } - _, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to update OAuth ClusterRoleBinding") + _, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update OAuth ClusterRoleBinding") + return ctrl.Result{RequeueAfter: requeueTime}, err + } + + if isLocalInteractive { + logger.Info("Creating RayClient Route") + _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredRayClientRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update RayClient Route") + return ctrl.Result{RequeueAfter: requeueTime}, err + } + } + + } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !isOpenShift { + logger.Info("Creating Dashboard Ingress") + _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, ingressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + // This log is info level since errors are not fatal and are expected + logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true) + return ctrl.Result{RequeueAfter: requeueTime}, err + } + if isLocalInteractive && ingressDomain != "" { + logger.Info("Creating RayClient Ingress") + _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, ingressDomain), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update RayClient Ingress") + return ctrl.Result{RequeueAfter: requeueTime}, err + } + } } return ctrl.Result{}, nil @@ -193,19 +235,23 @@ func desiredServiceAccount(cluster *rayv1.RayCluster) *coreapply.ServiceAccountA WithAnnotations(map[string]string{ "serviceaccounts.openshift.io/oauth-redirectreference.first": "" + `{"kind":"OAuthRedirectReference","apiVersion":"v1",` + - `"reference":{"kind":"Route","name":"` + routeNameFromCluster(cluster) + `"}}`, + `"reference":{"kind":"Route","name":"` + dashboardNameFromCluster(cluster) + `"}}`, }). WithOwnerReferences( v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) } -func routeNameFromCluster(cluster *rayv1.RayCluster) string { +func dashboardNameFromCluster(cluster *rayv1.RayCluster) string { return "ray-dashboard-" + cluster.Name } +func rayClientNameFromCluster(cluster *rayv1.RayCluster) string { + return "rayclient-" + cluster.Name +} + func desiredClusterRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration { - return routeapply.Route(routeNameFromCluster(cluster), cluster.Namespace). + return routeapply.Route(dashboardNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithSpec(routeapply.RouteSpec(). WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(oauthServiceNameFromCluster(cluster))). diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go new file mode 100644 index 00000000..348cd03d --- /dev/null +++ b/pkg/controllers/support.go @@ -0,0 +1,181 @@ +package controllers + +import ( + "context" + "fmt" + "strconv" + "strings" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + v1 "k8s.io/client-go/applyconfigurations/meta/v1" + networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + + routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" +) + +func serviceNameFromCluster(cluster *rayv1.RayCluster) string { + return cluster.Name + "-head-svc" +} + +func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration { + return routeapply.Route(rayClientNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithSpec(routeapply.RouteSpec(). + WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace). + WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(serviceNameFromCluster(cluster)).WithWeight(100)). + WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString("client"))). + WithTLS(routeapply.TLSConfig().WithTermination("passthrough")), + ). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} + +// Create an Ingress object for the RayCluster +func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressDomain string) *networkingv1ac.IngressApplyConfiguration { + return networkingv1ac.Ingress(rayClientNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithAnnotations(map[string]string{ + "nginx.ingress.kubernetes.io/rewrite-target": "/", + "nginx.ingress.kubernetes.io/ssl-redirect": "true", + "nginx.ingress.kubernetes.io/ssl-passthrough": "true", + }). + WithOwnerReferences(v1.OwnerReference(). + WithAPIVersion(cluster.APIVersion). + WithKind(cluster.Kind). + WithName(cluster.Name). + WithUID(types.UID(cluster.UID))). + WithSpec(networkingv1ac.IngressSpec(). + WithIngressClassName("nginx"). + WithRules(networkingv1ac.IngressRule(). + WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace + "." + ingressDomain). + WithHTTP(networkingv1ac.HTTPIngressRuleValue(). + WithPaths(networkingv1ac.HTTPIngressPath(). + WithPath("/"). + WithPathType(networkingv1.PathTypeImplementationSpecific). + WithBackend(networkingv1ac.IngressBackend(). + WithService(networkingv1ac.IngressServiceBackend(). + WithName(serviceNameFromCluster(cluster)). + WithPort(networkingv1ac.ServiceBackendPort(). + WithNumber(10001), + ), + ), + ), + ), + ), + ), + ) +} + +// Create an Ingress object for the RayCluster +func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration { + return networkingv1ac.Ingress(dashboardNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithOwnerReferences(v1.OwnerReference(). + WithAPIVersion(cluster.APIVersion). + WithKind(cluster.Kind). + WithName(cluster.Name). + WithUID(types.UID(cluster.UID))). + WithSpec(networkingv1ac.IngressSpec(). + WithRules(networkingv1ac.IngressRule(). + WithHost(ingressHost). // KinD hostname or ingressDomain + WithHTTP(networkingv1ac.HTTPIngressRuleValue(). + WithPaths(networkingv1ac.HTTPIngressPath(). + WithPath("/"). + WithPathType(networkingv1.PathTypePrefix). + WithBackend(networkingv1ac.IngressBackend(). + WithService(networkingv1ac.IngressServiceBackend(). + WithName(serviceNameFromCluster(cluster)). + WithPort(networkingv1ac.ServiceBackendPort(). + WithName(ingressServicePortName), + ), + ), + ), + ), + ), + ), + ) +} + +// isOnKindCluster checks if the current cluster is a KinD cluster. +// It searches for a node with a label commonly used by KinD clusters. +func isOnKindCluster(clientset *kubernetes.Clientset) (bool, error) { + nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: "kubernetes.io/hostname=kind-control-plane", + }) + if err != nil { + return false, err + } + // If we find one or more nodes with the label, assume it's a KinD cluster. + return len(nodes.Items) > 0, nil +} + +// getDiscoveryClient returns a discovery client for the current reconciler +func getDiscoveryClient(config *rest.Config) (*discovery.DiscoveryClient, error) { + return discovery.NewDiscoveryClientForConfig(config) +} + +// Check where we are running. We are trying to distinguish here whether +// this is vanilla kubernetes cluster or Openshift +func getClusterType(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressDomain string) (bool, string) { + // The discovery package is used to discover APIs supported by a Kubernetes API server. + logger := ctrl.LoggerFrom(ctx) + config, err := ctrl.GetConfig() + if err != nil && config == nil { + logger.Info("Cannot retrieve config, assuming we're on Vanilla Kubernetes") + return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) + } + dclient, err := getDiscoveryClient(config) + if err != nil && dclient == nil { + logger.Info("Cannot retrieve a DiscoveryClient, assuming we're on Vanilla Kubernetes") + return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) + } + apiGroupList, err := dclient.ServerGroups() + if err != nil { + logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes") + return false, "" + } + for i := 0; i < len(apiGroupList.Groups); i++ { + if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") { + logger.Info("We detected being on OpenShift!") + return true, "" + } + } + onKind, _ := isOnKindCluster(clientset) + if onKind && ingressDomain == "" { + logger.Info("We detected being on a KinD cluster!") + return false, "kind" + } + logger.Info("We detected being on Vanilla Kubernetes!") + return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) +} + +func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { + if r.Config != nil && r.Config.KubeRay != nil && r.Config.KubeRay.RayDashboardOAuthEnabled != nil { + return *r.Config.KubeRay.RayDashboardOAuthEnabled + } + return true +} + +func annotationBoolVal(ctx context.Context, cluster *rayv1.RayCluster, annotation string, defaultValue bool) bool { + logger := ctrl.LoggerFrom(ctx) + val, exists := cluster.ObjectMeta.Annotations[annotation] + if !exists || val == "" { + return defaultValue + } + boolVal, err := strconv.ParseBool(val) + if err != nil { + logger.Error(err, "Could not convert annotation value to bool", "annotation", annotation, "value", val) + return defaultValue + } + return boolVal +} From be7ac07a9ebcc06b8c51e5fdb120f6155c265fbd Mon Sep 17 00:00:00 2001 From: Mark Campbell Date: Fri, 5 Apr 2024 17:02:35 +0100 Subject: [PATCH 315/377] Removed InstaScale & MCAD from release workflow (#492) --- .../workflows/project-codeflare-release.yml | 76 +------------------ .github/workflows/tag-and-build.yml | 19 ----- 2 files changed, 2 insertions(+), 93 deletions(-) diff --git a/.github/workflows/project-codeflare-release.yml b/.github/workflows/project-codeflare-release.yml index 50fc26ae..017c7b02 100644 --- a/.github/workflows/project-codeflare-release.yml +++ b/.github/workflows/project-codeflare-release.yml @@ -10,15 +10,9 @@ on: replaces: description: 'The previous operator semantic version that this release replaces (for example: v0.0.0)' required: true - mcad-version: - description: 'Version of multi-cluster-app-dispatcher to be released (for example: v0.0.0)' - required: true codeflare-sdk-version: description: 'Version of CodeFlare-SDK to be released (for example: v0.0.0)' required: true - instascale-version: - description: 'Version of InstaScale to be released (for example: v0.0.0)' - required: true kuberay-version: description: 'Tested version of KubeRay (for example: v0.0.0)' required: true @@ -48,79 +42,13 @@ jobs: echo "Below are the release parameters set for the workflow:" echo "Operator Version: ${{ github.event.inputs.operator-version }}" echo "Replaces: ${{ github.event.inputs.replaces }}" - echo "MCAD Version: ${{ github.event.inputs.mcad-version }}" echo "CodeFlare SDK Version: ${{ github.event.inputs.codeflare-sdk-version }}" - echo "InstaScale Version: ${{ github.event.inputs.instascale-version }}" echo "Tested KubeRay Version: ${{ github.event.inputs.kuberay-version }}" echo "Is Stable: ${{ github.event.inputs.is-stable }}" echo "CodeFlare Repository Organization: ${{ github.event.inputs.codeflare-repository-organization }}" echo "Quay Organization: ${{ github.event.inputs.quay-organization }}" echo "Community Operators Prod Organization: ${{ github.event.inputs.community-operators-prod-organization }}" - release-mcad: - needs: check-kuberay-version - runs-on: ubuntu-latest - - steps: - - name: Check if MCAD release does exist - run: | - status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/${{ github.event.inputs.mcad-version }}) - if [[ "$status_code" == "200" ]]; then - echo "MCAD release with version ${{ github.event.inputs.mcad-version }} already exist. Will not create MCAD release." - fi - echo "MCAD_RELEASE_STATUS_CODE=$status_code" >> $GITHUB_ENV - - - name: Release MCAD - run: | - gh workflow run mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --ref go-1.20 --field tag=${{ github.event.inputs.mcad-version }} - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - shell: bash - if: ${{ env.MCAD_RELEASE_STATUS_CODE != '200' }} - - - name: Wait for MCAD run to finish - run: | - # wait for a while for Run to be started - sleep 5 - run_id=$(gh run list --workflow mcad-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --limit 1 --json databaseId --jq .[].databaseId) - gh run watch ${run_id} --repo ${{ github.event.inputs.codeflare-repository-organization }}/multi-cluster-app-dispatcher --interval 10 --exit-status - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - shell: bash - if: ${{ env.MCAD_RELEASE_STATUS_CODE != '200' }} - - release-instascale: - needs: release-mcad - runs-on: ubuntu-latest - - steps: - - name: Check if Instascale release does exist - run: | - status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/project-codeflare/instascale/releases/tag/${{ github.event.inputs.instascale-version }}) - if [[ "$status_code" == "200" ]]; then - echo "Instascale release with version ${{ github.event.inputs.instascale-version }} already exist. Will not create Instascale release." - fi - echo "INSTASCALE_RELEASE_STATUS_CODE=$status_code" >> $GITHUB_ENV - - - name: Release InstaScale - run: | - gh workflow run instascale-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --ref ${{ github.ref }} --field tag=${{ github.event.inputs.instascale-version }} --field mcad-version=${{ github.event.inputs.mcad-version }} - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - shell: bash - if: ${{ env.INSTASCALE_RELEASE_STATUS_CODE != '200' }} - - - name: Wait for InstaScale run to finish - run: | - # wait for a while for Run to be started - sleep 5 - run_id=$(gh run list --workflow instascale-release.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --limit 1 --json databaseId --jq .[].databaseId) - gh run watch ${run_id} --repo ${{ github.event.inputs.codeflare-repository-organization }}/instascale --interval 10 --exit-status - env: - GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} - shell: bash - if: ${{ env.INSTASCALE_RELEASE_STATUS_CODE != '200' }} - release-codeflare-sdk: needs: check-kuberay-version runs-on: ubuntu-latest @@ -156,13 +84,13 @@ jobs: if: ${{ env.SDK_RELEASE_STATUS_CODE != '200' }} release-codeflare-operator: - needs: [release-mcad, release-instascale, release-codeflare-sdk] + needs: [release-codeflare-sdk] runs-on: ubuntu-latest steps: - name: Release CodeFlare operator run: | - gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field mcad-version=${{ github.event.inputs.mcad-version }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field instascale-version=${{ github.event.inputs.instascale-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} + gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }} env: GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} shell: bash diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 9dc290c4..870ac35c 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -12,18 +12,10 @@ on: description: 'The previous semantic version that this tag replaces.' required: true default: 'v0.0.0-dev' - mcad-version: - description: 'Published version of multi-cluster-app-dispatcher' - required: true - default: 'v0.0.0-dev' codeflare-sdk-version: description: 'Published version of CodeFlare-SDK' required: true default: 'v0.0.0-dev' - instascale-version: - description: 'Published version of InstaScale' - required: true - default: 'v0.0.0-dev' kuberay-version: description: 'Tested version of KubeRay' required: true @@ -90,18 +82,9 @@ jobs: - name: Adjust Compatibility Matrix in readme run: | sed -i -E "s|(.*CodeFlare Operator.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.version }}\2${{ github.event.inputs.version }}\3|" README.md - sed -i -E "s|(.*Multi-Cluster App Dispatcher.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.mcad-version }}\2${{ github.event.inputs.mcad-version }}\3|" README.md sed -i -E "s|(.*CodeFlare-SDK.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3|" README.md - sed -i -E "s|(.*InstaScale.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.instascale-version }}\2${{ github.event.inputs.instascale-version }}\3|" README.md sed -i -E "s|(.*KubeRay.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kuberay-version }}\2${{ github.event.inputs.kuberay-version }}\3|" README.md - - name: Adjust MCAD, SDK and InstaScale dependencies in the code - run: | - sed -i -E "s/(.*MCAD_VERSION \?= ).*/\1${{ github.event.inputs.mcad-version }}/" Makefile - sed -i -E "s/(.*MCAD_REF \?= ).*/\1release-\${MCAD_VERSION}/" Makefile - sed -i -E "s/(.*INSTASCALE_VERSION \?= ).*/\1${{ github.event.inputs.instascale-version }}/" Makefile - sed -i -E "s/(.*KUBERAY_VERSION \?= ).*/\1${{ github.event.inputs.kuberay-version }}/" Makefile - - name: Update image version in params.env run: | VERSION=${{ github.event.inputs.version }} perl -i -pe 's/:(.*)$/:$ENV{"VERSION"}/' config/manager/params.env @@ -192,8 +175,6 @@ jobs: env: VERSION: ${{ github.event.inputs.version }} PREVIOUS_VERSION: ${{ github.event.inputs.replaces }} - INSTASCALE_VERSION: ${{ github.event.inputs.instascale-version }} - MCAD_VERSION: ${{ github.event.inputs.mcad-version }} GH_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} OPERATORS_REPO_FORK_ORG: ${{ github.event.inputs.community-operators-prod-fork-organization }} OPERATORS_REPO_ORG: ${{ github.event.inputs.community-operators-prod-organization }} From c771aae4dc990526b8b3b4d3817805fa479bded2 Mon Sep 17 00:00:00 2001 From: David Grove Date: Fri, 5 Apr 2024 13:04:51 -0400 Subject: [PATCH 316/377] Clean removal of MCADv1 and Instascale (#501) * Clean removal of MCAD and Instascale * entirely remove the MCAD/Instascale config structs * untested port of odh ray test to non-appwrapper version --- Makefile | 18 - config/crd/crd-appwrapper.yml | 462 ------------------ config/crd/crd-quotasubtree.yml | 115 ----- config/crd/crd-schedulingspec.yml | 80 --- config/crd/kustomization.yaml | 3 - config/crd/mcad/kustomization.yaml | 4 - config/e2e/config.yaml | 8 - config/e2e/kustomization.yaml | 1 - go.mod | 46 +- go.sum | 117 +---- main.go | 41 -- pkg/config/config.go | 28 -- test/e2e/instascale_app_wrapper.go | 140 ------ test/e2e/instascale_machinepool_test.go | 82 ---- test/e2e/instascale_machineset_test.go | 59 --- test/e2e/instascale_nodepool_test.go | 68 --- test/e2e/mnist_pytorch_mcad_job_test.go | 175 ------- ...est.go => mnist_rayjob_raycluster_test.go} | 50 +- test/odh/mcad_ray_test.go | 39 +- test/odh/pytorch_mcad_test.go | 77 --- test/upgrade/olm_upgrade_test.go | 216 -------- 21 files changed, 39 insertions(+), 1790 deletions(-) delete mode 100644 config/crd/crd-appwrapper.yml delete mode 100644 config/crd/crd-quotasubtree.yml delete mode 100644 config/crd/crd-schedulingspec.yml delete mode 100644 config/crd/mcad/kustomization.yaml delete mode 100644 config/e2e/config.yaml delete mode 100644 test/e2e/instascale_app_wrapper.go delete mode 100644 test/e2e/instascale_machinepool_test.go delete mode 100644 test/e2e/instascale_machineset_test.go delete mode 100644 test/e2e/instascale_nodepool_test.go delete mode 100644 test/e2e/mnist_pytorch_mcad_job_test.go rename test/e2e/{mnist_rayjob_mcad_raycluster_test.go => mnist_rayjob_raycluster_test.go} (81%) delete mode 100644 test/odh/pytorch_mcad_test.go delete mode 100644 test/upgrade/olm_upgrade_test.go diff --git a/Makefile b/Makefile index d2cd22d0..e1b77a8e 100644 --- a/Makefile +++ b/Makefile @@ -11,16 +11,6 @@ PREVIOUS_VERSION ?= v0.0.0-dev VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) -# INSTASCALE_VERSION defines the default version of the InstaScale controller -INSTASCALE_VERSION ?= v0.4.0 -INSTASCALE_REPO ?= github.com/project-codeflare/instascale - -# MCAD_VERSION defines the default version of the MCAD controller -MCAD_VERSION ?= v1.40.0 -MCAD_REPO ?= github.com/project-codeflare/multi-cluster-app-dispatcher -# Upstream MCAD is currently only creating release tags of the form `vX.Y.Z` (i.e the version) -MCAD_CRD ?= ${MCAD_REPO}/config/crd?ref=${MCAD_VERSION} - # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) KUBERAY_VERSION ?= v1.0.0 @@ -139,9 +129,6 @@ help: ## Display this help. .PHONY: manifests manifests: controller-gen kustomize install-yq ## Generate RBAC objects. $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml - $(KUSTOMIZE) build config/crd/mcad | $(YQ) -s '"crd-" + .spec.names.singular' --no-doc - mv crd-*.yml config/crd .PHONY: fmt fmt: ## Run go fmt against code. @@ -156,8 +143,6 @@ vet: ## Run go vet against code. .PHONY: modules modules: ## Update Go dependencies. - go get $(MCAD_REPO)@$(MCAD_VERSION) - go get $(INSTASCALE_REPO)@$(INSTASCALE_VERSION) go get github.com/ray-project/kuberay/ray-operator@$(KUBERAY_VERSION) go mod tidy @@ -166,8 +151,6 @@ build: fmt vet ## Build manager binary. go build \ -ldflags " \ -X 'main.OperatorVersion=$(BUILD_VERSION)' \ - -X 'main.McadVersion=$(MCAD_VERSION)' \ - -X 'main.InstaScaleVersion=$(INSTASCALE_VERSION)' \ -X 'main.BuildDate=$(BUILD_DATE)' \ " \ -o bin/manager main.go @@ -208,7 +191,6 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in .PHONY: undeploy undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/${ENV} | kubectl delete --ignore-not-found=$(ignore-not-found) -f - git restore config/* diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml deleted file mode 100644 index 9140857a..00000000 --- a/config/crd/crd-appwrapper.yml +++ /dev/null @@ -1,462 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: appwrappers.workload.codeflare.dev -spec: - group: workload.codeflare.dev - names: - kind: AppWrapper - listKind: AppWrapperList - plural: appwrappers - singular: appwrapper - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: Definition of AppWrapper class - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: AppWrapperSpec describes how the App Wrapper will look like. - properties: - priority: - format: int32 - type: integer - priorityslope: - format: float - type: number - resources: - description: a collection of AppWrapperResource - properties: - GenericItems: - items: - description: AppWrapperGenericResource is App Wrapper aggregation resource - properties: - allocated: - description: The number of allocated replicas from this resource type - format: int32 - type: integer - completionstatus: - description: 'Optional field that drives completion status of this AppWrapper. This field within an item of an AppWrapper determines the full state of the AppWrapper. The completionstatus field contains a list of conditions that make the associate item considered completed, for instance: - completion conditions could be "Complete" or "Failed". The associated item''s level .status.conditions[].type field is monitored for any one of these conditions. Once all items with this option is set and the conditionstatus is met the entire AppWrapper state will be changed to one of the valid AppWrapper completion state. Note: - this is an AND operation for all items where this option is set. See the list of AppWrapper states for a list of valid complete states.' - type: string - custompodresources: - description: Optional section that specifies resource requirements for non-standard k8s resources, follows same format as that of standard k8s resources. - items: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: ResourceList is a set of (resource name, quantity) pairs. - type: object - replicas: - type: integer - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'todo: replace with Containers []Container Contain v1.ResourceRequirements' - type: object - required: - - replicas - - requests - type: object - type: array - generictemplate: - description: The template for the resource; it is now a raw text because we don't know for what resource it should be instantiated - type: object - x-kubernetes-embedded-resource: true - x-kubernetes-preserve-unknown-fields: true - minavailable: - description: The minimal available pods to run for this AppWrapper; the default value is nil - format: int32 - type: integer - priority: - description: The priority of this resource - format: int32 - type: integer - priorityslope: - description: The increasing rate of priority value for this resource - format: float - type: number - replicas: - description: Replicas is the number of desired replicas - format: int32 - type: integer - type: object - type: array - type: object - schedulingSpec: - description: SchedSpec specifies the parameters used for scheduling generic items wrapped inside AppWrappers. It defines the policy for requeuing jobs based on the number of running pods. - properties: - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds. - properties: - expected: - type: integer - limit: - type: integer - overrun: - type: boolean - type: object - minAvailable: - description: Expected number of pods in running and/or completed state. Requeuing is triggered when the number of running/completed pods is not equal to this value. When not specified, requeuing is disabled and no check is performed. - type: integer - nodeSelector: - additionalProperties: - type: string - type: object - requeuing: - description: Specification of the requeuing strategy based on waiting time. Values in this field control how often the pod check should happen, and if requeuing has reached its maximum number of times. - properties: - growthType: - default: exponential - description: Growth strategy to increase the waiting time between requeuing checks. The values available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. - type: string - initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users cannot set this as it is taken from 'timeInSeconds'. - type: integer - maxNumRequeuings: - default: 0 - description: Maximum number of requeuing events allowed. Once this value is reached (e.g., 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic items are stopped and removed from the cluster (AppWrapper remains deployed). - type: integer - maxTimeInSeconds: - default: 0 - description: Maximum waiting time for requeuing checks. - type: integer - numRequeuings: - default: 0 - description: Field to keep track of how many times a requeuing event has been triggered. - type: integer - timeInSeconds: - default: 300 - description: Initial waiting time before requeuing conditions are checked. This value is specified by the user, but it may grow as requeuing events happen. - type: integer - type: object - type: object - selector: - description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - service: - description: AppWrapperService is App Wrapper service definition - properties: - spec: - description: ServiceSpec describes the attributes that a user creates on a service. - properties: - allocateLoadBalancerNodePorts: - description: allocateLoadBalancerNodePorts defines if NodePorts will be automatically allocated for services with type LoadBalancer. Default is "true". It may be set to "false" if the cluster load-balancer does not rely on NodePorts. If the caller requests specific NodePorts (by specifying a value), those requests will be respected, regardless of this field. This field may only be set for services with type LoadBalancer and will be cleared if the type is changed to any other type. - type: boolean - clusterIP: - description: 'clusterIP is the IP address of the service and is usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be blank) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are "None", empty string (""), or a valid IP address. Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - type: string - clusterIPs: - description: "ClusterIPs is a list of IP addresses assigned to this service, and are usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be empty) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are \"None\", empty string (\"\"), or a valid IP address. Setting this to \"None\" makes a \"headless service\" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. If this field is not specified, it will be initialized from the clusterIP field. If this field is specified, clients must ensure that clusterIPs[0] and clusterIP have the same value. \n This field may hold a maximum of two entries (dual-stack IPs, in either order). These IPs must correspond to the values of the ipFamilies field. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" - items: - type: string - type: array - x-kubernetes-list-type: atomic - externalIPs: - description: externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system. - items: - type: string - type: array - externalName: - description: externalName is the external reference that discovery mechanisms will return as an alias for this service (e.g. a DNS CNAME record). No proxying will be involved. Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". - type: string - externalTrafficPolicy: - description: externalTrafficPolicy describes how nodes distribute service traffic they receive on one of the Service's "externally-facing" addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure the service in a way that assumes that external load balancers will take care of balancing the service traffic between nodes, and so each node will deliver traffic only to the node-local endpoints of the service, without masquerading the client source IP. (Traffic mistakenly sent to a node with no endpoints will be dropped.) The default value, "Cluster", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). Note that traffic sent to an External IP or LoadBalancer IP from within the cluster will always get "Cluster" semantics, but clients sending to a NodePort from within the cluster may need to take traffic policy into account when picking a node. - type: string - healthCheckNodePort: - description: healthCheckNodePort specifies the healthcheck nodePort for the service. This only applies when type is set to LoadBalancer and externalTrafficPolicy is set to Local. If a value is specified, is in-range, and is not in use, it will be used. If not specified, a value will be automatically allocated. External systems (e.g. load-balancers) can use this port to determine if a given node holds endpoints for this service or not. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type). This field cannot be updated once set. - format: int32 - type: integer - internalTrafficPolicy: - description: InternalTrafficPolicy describes how nodes distribute service traffic they receive on the ClusterIP. If set to "Local", the proxy will assume that pods only want to talk to endpoints of the service on the same node as the pod, dropping the traffic if there are no local endpoints. The default value, "Cluster", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). - type: string - ipFamilies: - description: "IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this service. This field is usually assigned automatically based on cluster configuration and the ipFamilyPolicy field. If this field is specified manually, the requested family is available in the cluster, and ipFamilyPolicy allows it, it will be used; otherwise creation of the service will fail. This field is conditionally mutable: it allows for adding or removing a secondary IP family, but it does not allow changing the primary IP family of the Service. Valid values are \"IPv4\" and \"IPv6\". This field only applies to Services of types ClusterIP, NodePort, and LoadBalancer, and does apply to \"headless\" services. This field will be wiped when updating a Service to type ExternalName. \n This field may hold a maximum of two entries (dual-stack families, in either order). These families must correspond to the values of the clusterIPs field, if specified. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field." - items: - description: IPFamily represents the IP Family (IPv4 or IPv6). This type is used to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). - type: string - type: array - x-kubernetes-list-type: atomic - ipFamilyPolicy: - description: IPFamilyPolicy represents the dual-stack-ness requested or required by this Service. If there is no value provided, then this field will be set to SingleStack. Services can be "SingleStack" (a single IP family), "PreferDualStack" (two IP families on dual-stack configured clusters or a single IP family on single-stack clusters), or "RequireDualStack" (two IP families on dual-stack configured clusters, otherwise fail). The ipFamilies and clusterIPs fields depend on the value of this field. This field will be wiped when updating a service to type ExternalName. - type: string - loadBalancerClass: - description: loadBalancerClass is the class of the load balancer implementation this Service belongs to. If specified, the value of this field must be a label-style identifier, with an optional prefix, e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load balancer implementation is used, today this is typically done through the cloud provider integration, but should apply for any default implementation. If set, it is assumed that a load balancer implementation is watching for Services with a matching class. Any default load balancer implementation (e.g. cloud providers) should ignore Services that set this field. This field can only be set when creating or updating a Service to type 'LoadBalancer'. Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. - type: string - loadBalancerIP: - description: 'Only applies to Service Type: LoadBalancer. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature. Deprecated: This field was under-specified and its meaning varies across implementations, and it cannot support dual-stack. As of Kubernetes v1.24, users are encouraged to use implementation-specific annotations when available. This field may be removed in a future API version.' - type: string - loadBalancerSourceRanges: - description: 'If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' - items: - type: string - type: array - ports: - description: 'The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - items: - description: ServicePort contains information on service's port. - properties: - appProtocol: - description: The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and https://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. - type: string - name: - description: The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service. - type: string - nodePort: - description: 'The port on each node on which this service is exposed when type is NodePort or LoadBalancer. Usually assigned by the system. If a value is specified, in-range, and not in use it will be used, otherwise the operation will fail. If not specified, a port will be allocated if this Service requires one. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' - format: int32 - type: integer - port: - description: The port that will be exposed by this service. - format: int32 - type: integer - protocol: - default: TCP - description: The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". Default is TCP. - type: string - targetPort: - anyOf: - - type: integer - - type: string - description: 'Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod''s container ports. If this is not specified, the value of the ''port'' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' - x-kubernetes-int-or-string: true - required: - - port - type: object - type: array - x-kubernetes-list-map-keys: - - port - - protocol - x-kubernetes-list-type: map - publishNotReadyAddresses: - description: publishNotReadyAddresses indicates that any agent which deals with endpoints for this Service should disregard any indications of ready/not-ready. The primary use case for setting this field is for a StatefulSet's Headless Service to propagate SRV DNS records for its Pods for the purpose of peer discovery. The Kubernetes controllers that generate Endpoints and EndpointSlice resources for Services interpret this to mean that all endpoints are considered "ready" even if the Pods themselves are not. Agents which consume only Kubernetes generated endpoints through the Endpoints or EndpointSlice resources can safely assume this behavior. - type: boolean - selector: - additionalProperties: - type: string - description: 'Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' - type: object - x-kubernetes-map-type: atomic - sessionAffinity: - description: 'Supports "ClientIP" and "None". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - type: string - sessionAffinityConfig: - description: sessionAffinityConfig contains the configurations of session affinity. - properties: - clientIP: - description: clientIP contains the configurations of Client IP based session affinity. - properties: - timeoutSeconds: - description: timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". Default value is 10800(for 3 hours). - format: int32 - type: integer - type: object - type: object - type: - description: 'type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. "ClusterIP" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object or EndpointSlice objects. If clusterIP is "None", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a virtual IP. "NodePort" builds on ClusterIP and allocates a port on every node which routes to the same endpoints as the clusterIP. "LoadBalancer" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the same endpoints as the clusterIP. "ExternalName" aliases this service to the specified externalName. Several other fields do not apply to ExternalName services. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' - type: string - type: object - required: - - spec - type: object - required: - - resources - type: object - status: - description: AppWrapperStatus represents the current state of a AppWrapper - properties: - Succeeded: - description: The number of resources which reached phase Succeeded. - format: int32 - type: integer - canrun: - description: Can run? - type: boolean - conditions: - description: Represents the latest available observations of the AppWrapper's current condition. - items: - description: AppWrapperCondition describes the state of an AppWrapper at a certain point. - properties: - lastTransitionMicroTime: - description: Last time the condition transitioned from one status to another. - format: date-time - type: string - lastUpdateMicroTime: - description: The last time this condition was updated. - format: date-time - type: string - message: - description: A human-readable message indicating details about the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of AppWrapper condition. - type: string - required: - - status - - type - type: object - type: array - controllerfirstdispatchtimestamp: - description: Microsecond level timestamp when controller first dispatches the AppWrapper - format: date-time - type: string - controllerfirsttimestamp: - description: Microsecond level timestamp when controller first sees QueueJob (by Informer) - format: date-time - type: string - failed: - description: The number of resources which reached phase Failed. - format: int32 - type: integer - filterignore: - description: Tell Informer to ignore this update message (do not generate a controller event) - type: boolean - isdispatched: - description: Is Dispatched? - type: boolean - local: - description: Indicate if message is a duplicate (for Informer to recognize duplicate messages) - type: boolean - message: - type: string - numberOfRequeueings: - description: Field to keep track of how many times a requeuing event has been triggered - type: integer - pending: - description: The number of pending pods. - format: int32 - type: integer - pendingpodconditions: - description: Represents the latest available observations of pods belonging to the AppWrapper. - items: - properties: - conditions: - items: - description: PodCondition contains details for the current condition of this pod. - properties: - lastProbeTime: - description: Last time we probed the condition. - format: date-time - type: string - lastTransitionTime: - description: Last time the condition transitioned from one status to another. - format: date-time - type: string - message: - description: Human-readable message indicating details about last transition. - type: string - reason: - description: Unique, one-word, CamelCase reason for the condition's last transition. - type: string - status: - description: 'Status is the status of the condition. Can be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' - type: string - type: - description: 'Type is the type of the condition. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' - type: string - required: - - status - - type - type: object - type: array - podname: - type: string - type: object - type: array - queuejobstate: - description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, ... - type: string - requeueingTimeInSeconds: - description: Field to keep track of total number of seconds spent in requeueing - type: integer - running: - format: int32 - type: integer - sender: - description: Indicate sender of this message (extremely useful for debugging) - type: string - state: - description: State - Pending, Running, Failed, Deleted - type: string - systempriority: - description: System defined Priority - format: float - type: number - template: - description: The minimal available resources to run for this AppWrapper (is this different from the MinAvailable from JobStatus) - format: int32 - type: integer - totalcpu: - description: The number of CPU consumed by all pods belonging to the AppWrapper. - format: int32 - type: integer - totalgpu: - description: The total number of GPUs consumed by all pods belonging to the AppWrapper. - format: int32 - type: integer - totalmemory: - description: The amount of memory consumed by all pods belonging to the AppWrapper. - format: int32 - type: integer - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/crd-quotasubtree.yml b/config/crd/crd-quotasubtree.yml deleted file mode 100644 index 924aaacd..00000000 --- a/config/crd/crd-quotasubtree.yml +++ /dev/null @@ -1,115 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: quotasubtrees.quota.codeflare.dev -spec: - group: quota.codeflare.dev - names: - kind: QuotaSubtree - listKind: QuotaSubtreeList - plural: quotasubtrees - singular: quotasubtree - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: QuotaSubtree is a specification for a quota subtree resource - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: QuotaSubtreeSpec is the spec for a resource plan - properties: - children: - items: - description: Child is the spec for a QuotaSubtree resource - properties: - name: - type: string - namespace: - type: string - path: - type: string - quotas: - description: Quota is the spec for a QuotaSubtree resource - properties: - disabled: - type: boolean - hardLimit: - type: boolean - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - type: object - type: array - parent: - type: string - parentNamespace: - type: string - type: object - status: - description: QuotaSubtreeStatus is the status for a QuotaSubtree resource - properties: - children: - items: - description: ResourceAllocation is the spec for the child status - properties: - allocated: - description: ResourceAllocationStatus is the spec for the child resource usage - properties: - requests: - additionalProperties: - type: string - type: object - type: object - name: - type: string - namespace: - type: string - path: - type: string - type: object - type: array - totalAllocation: - description: ResourceAllocation is the spec for the child status - properties: - allocated: - description: ResourceAllocationStatus is the spec for the child resource usage - properties: - requests: - additionalProperties: - type: string - type: object - type: object - name: - type: string - namespace: - type: string - path: - type: string - type: object - required: - - children - - totalAllocation - type: object - required: - - spec - type: object - served: true - storage: true diff --git a/config/crd/crd-schedulingspec.yml b/config/crd/crd-schedulingspec.yml deleted file mode 100644 index 373dc89d..00000000 --- a/config/crd/crd-schedulingspec.yml +++ /dev/null @@ -1,80 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: schedulingspecs.workload.codeflare.dev -spec: - group: workload.codeflare.dev - names: - kind: SchedulingSpec - listKind: SchedulingSpecList - plural: schedulingspecs - singular: schedulingspec - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds. - properties: - expected: - type: integer - limit: - type: integer - overrun: - type: boolean - type: object - minAvailable: - description: Expected number of pods in running and/or completed state. Requeuing is triggered when the number of running/completed pods is not equal to this value. When not specified, requeuing is disabled and no check is performed. - type: integer - nodeSelector: - additionalProperties: - type: string - type: object - requeuing: - description: Specification of the requeuing strategy based on waiting time. Values in this field control how often the pod check should happen, and if requeuing has reached its maximum number of times. - properties: - growthType: - default: exponential - description: Growth strategy to increase the waiting time between requeuing checks. The values available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. - type: string - initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users cannot set this as it is taken from 'timeInSeconds'. - type: integer - maxNumRequeuings: - default: 0 - description: Maximum number of requeuing events allowed. Once this value is reached (e.g., 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic items are stopped and removed from the cluster (AppWrapper remains deployed). - type: integer - maxTimeInSeconds: - default: 0 - description: Maximum waiting time for requeuing checks. - type: integer - numRequeuings: - default: 0 - description: Field to keep track of how many times a requeuing event has been triggered. - type: integer - timeInSeconds: - default: 300 - description: Initial waiting time before requeuing conditions are checked. This value is specified by the user, but it may grow as requeuing events happen. - type: integer - type: object - type: object - required: - - metadata - type: object - served: true - storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index faaff445..3b60faba 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,8 +2,5 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- crd-appwrapper.yml -- crd-quotasubtree.yml -- crd-schedulingspec.yml #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml deleted file mode 100644 index bbf6141c..00000000 --- a/config/crd/mcad/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.40.0 diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml deleted file mode 100644 index 4b01a4b0..00000000 --- a/config/e2e/config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: codeflare-operator-config -data: - config.yaml: | - mcad: - enabled: true diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml index 772370da..e06123b2 100644 --- a/config/e2e/kustomization.yaml +++ b/config/e2e/kustomization.yaml @@ -1,7 +1,6 @@ namespace: openshift-operators bases: -- config.yaml - ../default patches: diff --git a/go.mod b/go.mod index 93fa2c9f..80f3a52b 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,6 @@ require ( github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a - github.com/project-codeflare/instascale v0.4.0 - github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0 github.com/ray-project/kuberay/ray-operator v1.0.0 go.uber.org/zap v1.26.0 k8s.io/api v0.27.8 @@ -25,25 +23,16 @@ require ( replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 require ( - github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect - github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/eapache/go-resiliency v1.3.0 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.4 // indirect - github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect @@ -54,53 +43,32 @@ require ( github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/cel-go v0.12.7 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/microcosm-cc/bluemonday v1.0.18 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stoewer/go-strcase v1.2.0 // indirect - go.etcd.io/etcd/api/v3 v3.5.7 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect - go.etcd.io/etcd/client/v3 v3.5.7 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect - go.opentelemetry.io/otel v1.19.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect - go.opentelemetry.io/otel/metric v1.19.0 // indirect - go.opentelemetry.io/otel/sdk v1.19.0 // indirect - go.opentelemetry.io/otel/trace v1.19.0 // indirect - go.opentelemetry.io/proto/otlp v1.0.0 // indirect + github.com/stretchr/testify v1.8.4 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.18.0 // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -108,22 +76,12 @@ require ( golang.org/x/tools v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect - google.golang.org/grpc v1.58.3 // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.27.7 // indirect - k8s.io/apiserver v0.27.8 // indirect - k8s.io/kms v0.27.8 // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect - k8s.io/metrics v0.26.2 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect - sigs.k8s.io/custom-metrics-apiserver v0.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index 7d5f4c95..51ec31f4 100644 --- a/go.sum +++ b/go.sum @@ -13,15 +13,12 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -34,12 +31,9 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -47,10 +41,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= -github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -58,8 +48,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -77,24 +65,15 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.4.0 h1:y9YHcjnjynCd/DVbg5j9L/33jQM3MxJlbj/zWskzfGU= -github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= -github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -103,13 +82,10 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -126,11 +102,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -143,7 +116,6 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -187,9 +159,6 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/cel-go v0.12.7 h1:jM6p55R0MKBg79hZjn1zs2OlrywZ1Vk00rxVvad1/O0= -github.com/google/cel-go v0.12.7/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -226,18 +195,7 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -245,8 +203,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= @@ -300,7 +256,6 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -345,8 +300,6 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= -github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -359,12 +312,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= @@ -389,10 +340,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a h1:Yk9J5qXjp+yfSRCzS0EElrhpTgfYJ+S+W/z84cmlmX4= github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= -github.com/project-codeflare/instascale v0.4.0 h1:l/cb+x4FrJ2bN9wXjv1mCngy77tVw0CLMiqJovTAflo= -github.com/project-codeflare/instascale v0.4.0/go.mod h1:CpduFXKeuqYW4Ph1CPOJV6dpAdpebOxhbU4CmccZWSo= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0 h1:IkTmd/W/zxcsC5s4EbnW74PFpkQVEiTc/8rWWwFw0Ok= -github.com/project-codeflare/multi-cluster-app-dispatcher v1.40.0/go.mod h1:XCZKkq8Mz2WySbV3NfVINNciy+as7Rq9Xs2megNFbdk= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 h1:oyhdLdc4BgA4zcH1zlRrSrYpzuVxV5QLDbyIXrwnQqs= +github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -427,7 +376,6 @@ github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDN github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -435,14 +383,9 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -459,11 +402,10 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -471,41 +413,12 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY= -go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= -go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg= -go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY= -go.etcd.io/etcd/client/v2 v2.305.7 h1:AELPkjNR3/igjbO7CjyF1fPuVPjrblliiKj+Y6xSGOU= -go.etcd.io/etcd/client/v3 v3.5.7 h1:u/OhpiuCgYY8awOHlhIhmGIGpxfBU/GZBUP3m/3/Iz4= -go.etcd.io/etcd/client/v3 v3.5.7/go.mod h1:sOWmj9DZUMyAngS7QQwCyAXXAL6WhgTOPLNS/NabQgw= -go.etcd.io/etcd/pkg/v3 v3.5.7 h1:obOzeVwerFwZ9trMWapU/VjDcYUJb5OfgC1zqEGWO/0= -go.etcd.io/etcd/raft/v3 v3.5.7 h1:aN79qxLmV3SvIq84aNTliYGmjwsW6NqJSnqmI1HLJKc= -go.etcd.io/etcd/server/v3 v3.5.7 h1:BTBD8IJUV7YFgsczZMHhMTS67XuA4KpRquL0MFOJGRk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0 h1:RsQi0qJ2imFfCvZabqzM9cNXBG8k6gXMv1A0cXRmH6A= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0/go.mod h1:vsh3ySueQCiKPxFLvjWC4Z135gIa34TQ/NSqkDTZYUM= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= -go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= -go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= -go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= -go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= -go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= -go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= -go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -540,7 +453,6 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -637,8 +549,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -833,12 +743,6 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= -google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= -google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= -google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -854,8 +758,6 @@ google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -882,9 +784,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -913,31 +812,21 @@ k8s.io/apiextensions-apiserver v0.27.7 h1:YqIOwZAUokzxJIjunmUd4zS1v3JhK34EPXn+pP k8s.io/apiextensions-apiserver v0.27.7/go.mod h1:x0p+b5a955lfPz9gaDeBy43obM12s+N9dNHK6+dUL+g= k8s.io/apimachinery v0.27.8 h1:Xg+ogjDm8s7KmV3vZGf7uOZ0jrC6FPy2Lk/h7BIRmvg= k8s.io/apimachinery v0.27.8/go.mod h1:EIXLxLt/b1muPITiF5zlrcm7I+YnXsIgM+0GdnPTQvA= -k8s.io/apiserver v0.27.8 h1:fdrEm98wl1lxFoF8tCdQiYQOfmU3TOGkQr4YTLYHwzc= -k8s.io/apiserver v0.27.8/go.mod h1:8yLtDnOdRq9fubaea0zQ3AVheQUinb5Y/RqBUdQMT0Y= k8s.io/client-go v0.27.8 h1:uXIsGniqc85kKQ8FV0iAwspb0JHtS1UybqrqzHaQ9hc= k8s.io/client-go v0.27.8/go.mod h1:Ka6MUpObn3LRTfFPvjzyettp8LXCbhqLzZfi8TD4fP8= k8s.io/component-base v0.27.8 h1:O8YRFv/wWvoo9z62p1N52lq+w5FpzILAlE1h8b9o3K8= k8s.io/component-base v0.27.8/go.mod h1:h3uyZl+bFQeuLRz3owfSLaw3JKTrn6gmbvybkkW2z+I= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kms v0.27.8 h1:D7s2vZAFWKD1+1Yg6Y4C+l7k+ZjEsMc7BjY0QRKAOY8= -k8s.io/kms v0.27.8/go.mod h1:+yIM/3y+UgdaZSQaMJObNONp0i237t/dVrxswECh7a0= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= -k8s.io/metrics v0.26.2 h1:2gUvUWWnHPdE2tyA5DvyHC8HGryr+izhY9i5dzLP06s= -k8s.io/metrics v0.26.2/go.mod h1:PX1wm9REV9hSGuw9GcXTFNDgab1KRXck3mNeiLYbRho= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= sigs.k8s.io/controller-runtime v0.15.3 h1:L+t5heIaI3zeejoIyyvLQs5vTVu/67IU2FfisVzFlBc= sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds= -sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 h1:puQ5YlyBjhxg+OQ1YPMJXwtk7WhC4E6AlWIQ9pC8jws= -sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3/go.mod h1:9nUXR/EgdYZto1aQ6yhwOksPR7J979jSyOqic1IgaOo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/main.go b/main.go index 658b55e9..752f146f 100644 --- a/main.go +++ b/main.go @@ -24,12 +24,6 @@ import ( "strings" "time" - instascale "github.com/project-codeflare/instascale/controllers" - instascaleconfig "github.com/project-codeflare/instascale/pkg/config" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - quotasubtreev1alpha1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/quotaplugins/quotasubtree/v1alpha1" - mcadconfig "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config" - mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" "go.uber.org/zap/zapcore" @@ -52,8 +46,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/yaml" - configv1 "github.com/openshift/api/config/v1" - machinev1beta1 "github.com/openshift/api/machine/v1beta1" routev1 "github.com/openshift/api/route/v1" "github.com/project-codeflare/codeflare-operator/pkg/config" @@ -74,12 +66,6 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - // MCAD - utilruntime.Must(mcadv1beta1.AddToScheme(scheme)) - utilruntime.Must(quotasubtreev1alpha1.AddToScheme(scheme)) - // InstaScale - utilruntime.Must(configv1.Install(scheme)) - utilruntime.Must(machinev1beta1.Install(scheme)) // Ray utilruntime.Must(rayv1.AddToScheme(scheme)) // OpenShift Route @@ -127,15 +113,6 @@ func main() { }, LeaderElection: &configv1alpha1.LeaderElectionConfiguration{}, }, - MCAD: &config.MCADConfiguration{ - Enabled: pointer.Bool(false), - }, - InstaScale: &config.InstaScaleConfiguration{ - Enabled: pointer.Bool(false), - InstaScaleConfiguration: instascaleconfig.InstaScaleConfiguration{ - MaxScaleoutAllowed: 5, - }, - }, KubeRay: &config.KubeRayConfiguration{ RayDashboardOAuthEnabled: pointer.Bool(true), }, @@ -169,24 +146,6 @@ func main() { }) exitOnError(err, "unable to start manager") - if pointer.BoolDeref(cfg.MCAD.Enabled, false) { - mcadQueueController := mcad.NewJobController(mgr.GetConfig(), &cfg.MCAD.MCADConfiguration, &mcadconfig.MCADConfigurationExtended{}) - if mcadQueueController == nil { - // FIXME: update NewJobController so it follows Go idiomatic error handling and return an error instead of a nil object - os.Exit(1) - } - mcadQueueController.Run(ctx.Done()) - } - - if pointer.BoolDeref(cfg.InstaScale.Enabled, false) { - instaScaleController := &instascale.AppWrapperReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Config: cfg.InstaScale.InstaScaleConfiguration, - } - exitOnError(instaScaleController.SetupWithManager(context.Background(), mgr), "Error setting up InstaScale controller") - } - v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) if v { rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg} diff --git a/pkg/config/config.go b/pkg/config/config.go index 85a03385..124bb0eb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,9 +17,6 @@ limitations under the License. package config import ( - instascale "github.com/project-codeflare/instascale/pkg/config" - mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config" - configv1alpha1 "k8s.io/component-base/config/v1alpha1" ) @@ -31,12 +28,6 @@ type CodeFlareOperatorConfiguration struct { // ControllerManager returns the configurations for controllers ControllerManager `json:",inline"` - // The MCAD controller configuration - MCAD *MCADConfiguration `json:"mcad,omitempty"` - - // The InstaScale controller configuration - InstaScale *InstaScaleConfiguration `json:"instascale,omitempty"` - KubeRay *KubeRayConfiguration `json:"kuberay,omitempty"` } @@ -44,25 +35,6 @@ type KubeRayConfiguration struct { RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"` } -type MCADConfiguration struct { - // enabled controls whether the MCAD controller is started. - // It defaults to false. - Enabled *bool `json:"enabled,omitempty"` - - // The InstaScale controller configuration - mcad.MCADConfiguration `json:",inline,omitempty"` -} - -type InstaScaleConfiguration struct { - // enabled controls whether the InstaScale controller is started. - // It may default to true on platforms that InstaScale supports. - // Otherwise, it defaults to false. - Enabled *bool `json:"enabled,omitempty"` - - // The InstaScale controller configuration - instascale.InstaScaleConfiguration `json:",inline,omitempty"` -} - type ControllerManager struct { // Metrics contains the controller metrics configuration // +optional diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go deleted file mode 100644 index 45305dda..00000000 --- a/test/e2e/instascale_app_wrapper.go +++ /dev/null @@ -1,140 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func instaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) *mcadv1beta1.AppWrapper { - // Batch Job - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist", - Namespace: namespace.Name, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "job", - Image: GetPyTorchImage(), - Env: []corev1.EnvVar{ - {Name: "PYTHONUSERBASE", Value: "/workdir"}, - {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, - }, - Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, - Args: []string{"$PYTHONUSERBASE"}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - { - Name: "workdir", - MountPath: "/workdir", - }, - }, - WorkingDir: "/workdir", - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - { - Name: "workdir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - }, - }, - } - - // AppWrapper - aw := &mcadv1beta1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-instascale", - Namespace: namespace.Name, - Labels: map[string]string{ - "orderedinstance": "g4dn.xlarge", - }, - }, - Spec: mcadv1beta1.AppWrapperSpec{ - AggrResources: mcadv1beta1.AppWrapperResourceList{ - GenericItems: []mcadv1beta1.AppWrapperGenericResource{ - { - CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ - { - Replicas: 1, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - "nvidia.com/gpu": resource.MustParse("1"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("1G"), - "nvidia.com/gpu": resource.MustParse("1"), - }, - }, - { - Replicas: 1, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("1G"), - }, - }, - }, - GenericTemplate: Raw(test, job), - CompletionStatus: "Complete", - }, - }, - }, - }, - } - - return aw -} diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go deleted file mode 100644 index 64e53f8a..00000000 --- a/test/e2e/instascale_machinepool_test.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "fmt" - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestInstascaleMachinePool(t *testing.T) { - test := With(t) - test.T().Parallel() - - clusterType := GetClusterType(test) - if clusterType != OsdCluster { - test.T().Skipf("Skipping test as not running on an OSD cluster, resolved cluster type: %s", clusterType) - } - - namespace := test.NewTestNamespace() - - // Test configuration - cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }) - - //create OCM connection - connection := CreateOCMConnection(test) - defer connection.Close() - - // Setup batch job and AppWrapper - aw := instaScaleJobAppWrapper(test, namespace, cm) - - expectedLabel := fmt.Sprintf("%s-%s", aw.Name, aw.Namespace) - // check existing cluster machine pool resources - // look for a machine pool with a label key equal to aw.name-aw.namespace - expect NOT to find it - test.Expect(GetMachinePools(test, connection)). - ShouldNot(ContainElement(WithTransform(MachinePoolLabels, HaveKey(expectedLabel)))) - - // apply AppWrapper to cluster - _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) - - // assert that AppWrapper goes to "Running" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - - // look for a machine pool with a label key equal to aw.name-aw.namespace - expect to find it - test.Eventually(MachinePools(test, connection), TestTimeoutLong). - Should(ContainElement(WithTransform(MachinePoolLabels, HaveKey(expectedLabel)))) - - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - - // look for a machine pool with a label key equal to aw.name-aw.namespace - expect NOT to find it - test.Eventually(MachinePools(test, connection), TestTimeoutLong). - ShouldNot(ContainElement(WithTransform(MachinePoolLabels, HaveKey(expectedLabel)))) - -} diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go deleted file mode 100644 index 827fd404..00000000 --- a/test/e2e/instascale_machineset_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package e2e - -import ( - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestInstascaleMachineSet(t *testing.T) { - test := With(t) - test.T().Parallel() - - // skip test if not using machine sets - clusterType := GetClusterType(test) - if clusterType != OcpCluster { - test.T().Skipf("Skipping test as not running on an OCP cluster, resolved cluster type: %s", clusterType) - } - - namespace := test.NewTestNamespace() - - // Test configuration - cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }) - - // Setup batch job and AppWrapper - aw := instaScaleJobAppWrapper(test, namespace, cm) - - // look for machine set with aw name - expect to find it - test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal(aw.Name)))) - // look for machine belonging to the machine set, there should be none - test.Expect(GetMachines(test, aw.Name)).Should(BeEmpty()) - - // apply AppWrapper to cluster - _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - - // assert that AppWrapper goes to "Running" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - - // look for machine belonging to the machine set - expect to find it - test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(HaveLen(1)) - - // assert that the AppWrapper goes to "Completed" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - - // look for machine belonging to the machine set - there should be none - test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(BeEmpty()) - -} diff --git a/test/e2e/instascale_nodepool_test.go b/test/e2e/instascale_nodepool_test.go deleted file mode 100644 index 8721663d..00000000 --- a/test/e2e/instascale_nodepool_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package e2e - -import ( - "fmt" - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestInstascaleNodepool(t *testing.T) { - - test := With(t) - test.T().Parallel() - - clusterType := GetClusterType(test) - if clusterType != HypershiftCluster { - test.T().Skipf("Skipping test as not running on an Hypershift cluster, resolved cluster type: %s", clusterType) - } - - namespace := test.NewTestNamespace() - - // Test configuration - cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }) - - //create OCM connection - connection := CreateOCMConnection(test) - defer connection.Close() - - // Setup batch job and AppWrapper - aw := instaScaleJobAppWrapper(test, namespace, cm) - - expectedLabel := fmt.Sprintf("%s-%s", aw.Name, aw.Namespace) - // check existing cluster resources - // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect NOT to find it - test.Expect(GetNodePools(test, connection)). - ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) - - // apply AppWrapper to cluster - _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) - - // assert that AppWrapper goes to "Running" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - - // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect to find it - test.Eventually(NodePools(test, connection), TestTimeoutLong). - Should(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) - - // assert that the AppWrapper goes to "Completed" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - - // look for a node pool with a label key equal to aw.Name-aw.Namespace - expect NOT to find it - test.Eventually(NodePools(test, connection), TestTimeoutLong). - ShouldNot(ContainElement(WithTransform(NodePoolLabels, HaveKey(expectedLabel)))) - -} diff --git a/test/e2e/mnist_pytorch_mcad_job_test.go b/test/e2e/mnist_pytorch_mcad_job_test.go deleted file mode 100644 index a642d31c..00000000 --- a/test/e2e/mnist_pytorch_mcad_job_test.go +++ /dev/null @@ -1,175 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// Trains the MNIST dataset as a batch Job managed by MCAD, and asserts successful completion of the training job. -func TestMNISTPyTorchMCAD(t *testing.T) { - test := With(t) - test.T().Parallel() - - // Create a namespace - namespace := test.NewTestNamespace() - - // Test configuration - config := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist-mcad", - Namespace: namespace.Name, - }, - BinaryData: map[string][]byte{ - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }, - Immutable: Ptr(true), - } - config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - - // Batch Job - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist", - Namespace: namespace.Name, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "job", - Image: GetPyTorchImage(), - Env: []corev1.EnvVar{ - {Name: "PYTHONUSERBASE", Value: "/workdir"}, - {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, - {Name: "PIP_INDEX_URL", Value: GetPipIndexURL()}, - {Name: "PIP_TRUSTED_HOST", Value: GetPipTrustedHost()}, - }, - Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - { - Name: "workdir", - MountPath: "/workdir", - }, - }, - WorkingDir: "/workdir", - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - { - Name: "workdir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - }, - }, - } - - // Create an AppWrapper resource - aw := &mcadv1beta1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist", - Namespace: namespace.Name, - }, - Spec: mcadv1beta1.AppWrapperSpec{ - AggrResources: mcadv1beta1.AppWrapperResourceList{ - GenericItems: []mcadv1beta1.AppWrapperGenericResource{ - { - DesiredAvailable: 1, - CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ - { - Replicas: 1, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("1G"), - }, - }, - }, - GenericTemplate: Raw(test, job), - CompletionStatus: "Complete", - }, - }, - }, - }, - } - - _, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) - - test.T().Logf("Waiting for MCAD %s/%s to be running", aw.Namespace, aw.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutLong).Should( - Or( - WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)), - WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateFailed)), - )) - - // Assert the job has completed successfully - test.Expect(GetAppWrapper(test, namespace, aw.Name)). - To(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) -} diff --git a/test/e2e/mnist_rayjob_mcad_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go similarity index 81% rename from test/e2e/mnist_rayjob_mcad_raycluster_test.go rename to test/e2e/mnist_rayjob_raycluster_test.go index 1118079e..46f37d7a 100644 --- a/test/e2e/mnist_rayjob_mcad_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/gomega" . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" @@ -29,13 +28,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// Trains the MNIST dataset as a RayJob, executed by a Ray cluster managed by MCAD, -// and asserts successful completion of the training job. -func TestMNISTRayJobMCADRayCluster(t *testing.T) { +// Trains the MNIST dataset as a RayJob, executed by a Ray cluster +// directly managed by Kueue, and asserts successful completion of the training job. +func TestMNISTRayJobRayCluster(t *testing.T) { test := With(t) test.T().Parallel() - // Create a namespace + // Create a namespace and localqueue in that namespace namespace := test.NewTestNamespace() // MNIST training script @@ -172,44 +171,13 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) { }, } - // Create an AppWrapper resource - aw := &mcadv1beta1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ray-cluster", - Namespace: namespace.Name, - }, - Spec: mcadv1beta1.AppWrapperSpec{ - AggrResources: mcadv1beta1.AppWrapperResourceList{ - GenericItems: []mcadv1beta1.AppWrapperGenericResource{ - { - DesiredAvailable: 1, - CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ - { - Replicas: 2, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("100m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("1G"), - }, - }, - }, - GenericTemplate: Raw(test, rayCluster), - }, - }, - }, - }, - } - - aw, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) + _, err = test.Client().Ray().RayV1().RayClusters(namespace.Name).Create(test.Ctx(), rayCluster, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created MCAD %s/%s successfully", aw.Namespace, aw.Name) + test.T().Logf("Created RayCluster %s/%s successfully", rayCluster.Namespace, rayCluster.Name) - test.T().Logf("Waiting for MCAD %s/%s to be running", aw.Namespace, aw.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + test.T().Logf("Waiting for RayCluster %s/%s to be running", rayCluster.Namespace, rayCluster.Name) + test.Eventually(RayCluster(test, namespace.Name, rayCluster.Name), TestTimeoutMedium). + Should(WithTransform(RayClusterState, Equal(rayv1.Ready))) rayJob := &rayv1.RayJob{ TypeMeta: metav1.TypeMeta{ diff --git a/test/odh/mcad_ray_test.go b/test/odh/mcad_ray_test.go index b72456c5..967fc54c 100644 --- a/test/odh/mcad_ray_test.go +++ b/test/odh/mcad_ray_test.go @@ -20,11 +20,13 @@ import ( "testing" . "github.com/onsi/gomega" + gomega "github.com/onsi/gomega" . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestMCADRay(t *testing.T) { @@ -44,14 +46,9 @@ func TestMCADRay(t *testing.T) { // Create RBAC, retrieve token for user with limited rights policyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, { Verbs: []string{"get", "list"}, - APIGroups: []string{rayv1alpha1.GroupVersion.Group}, + APIGroups: []string{rayv1.GroupVersion.Group}, Resources: []string{"rayclusters", "rayclusters/status"}, }, { @@ -87,18 +84,17 @@ func TestMCADRay(t *testing.T) { // Create Notebook CR createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) - // Make sure the AppWrapper is created and running - test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). + // Make sure the RayCluster is created and running + test.Eventually(rayClusters(test, namespace), TestTimeoutLong). Should( And( HaveLen(1), - ContainElement(WithTransform(AppWrapperName, HavePrefix("mnisttest"))), - ContainElement(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))), + ContainElement(WithTransform(AppWrapperState, Equal(rayv1.Ready))), ), ) - // Make sure the AppWrapper finishes and is deleted - test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). + // Make sure the RayCluster finishes and is deleted + test.Eventually(rayClusters(test, namespace), TestTimeoutLong). Should(HaveLen(0)) } @@ -134,3 +130,18 @@ func readMnistPy(test Test) []byte { return ParseTemplate(test, template, props) } + +// TODO: This belongs on codeflare-common/support/ray.go +func rayClusters(t Test, namespace *corev1.Namespace) func(g gomega.Gomega) []*rayv1.RayCluster { + return func(g gomega.Gomega) []*rayv1.RayCluster { + rcs, err := t.Client().Ray().RayV1().RayClusters(namespace.Name).List(t.Ctx(), metav1.ListOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + rcsp := []*rayv1.RayCluster{} + for _, v := range rcs.Items { + rcsp = append(rcsp, &v) + } + + return rcsp + } +} diff --git a/test/odh/pytorch_mcad_test.go b/test/odh/pytorch_mcad_test.go deleted file mode 100644 index 0dd33a36..00000000 --- a/test/odh/pytorch_mcad_test.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package odh - -import ( - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - rbacv1 "k8s.io/api/rbac/v1" -) - -func TestMnistPyTorchMCAD(t *testing.T) { - test := With(t) - - // Create a namespace - namespace := test.NewTestNamespace() - - // Test configuration - jupyterNotebookConfigMapFileName := "mnist_mcad_mini.ipynb" - config := CreateConfigMap(test, namespace.Name, map[string][]byte{ - // MNIST MCAD Notebook - jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_mcad_mini.ipynb"), - }) - - // Create RBAC, retrieve token for user with limited rights - policyRules := []rbacv1.PolicyRule{ - { - Verbs: []string{"get", "create", "delete", "list", "patch", "update"}, - APIGroups: []string{mcadv1beta1.GroupName}, - Resources: []string{"appwrappers"}, - }, - // Needed for job.logs() - { - Verbs: []string{"get"}, - APIGroups: []string{""}, - Resources: []string{"pods/log"}, - }, - } - sa := CreateServiceAccount(test, namespace.Name) - role := CreateRole(test, namespace.Name, policyRules) - CreateRoleBinding(test, namespace.Name, sa, role) - token := CreateToken(test, namespace.Name, sa) - - // Create Notebook CR - createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName) - - // Make sure the AppWrapper is created and running - test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). - Should( - And( - HaveLen(1), - ContainElement(WithTransform(AppWrapperName, HavePrefix("mnistjob"))), - ContainElement(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))), - ), - ) - - // Make sure the AppWrapper finishes and is deleted - test.Eventually(AppWrappers(test, namespace), TestTimeoutLong). - Should(HaveLen(0)) -} diff --git a/test/upgrade/olm_upgrade_test.go b/test/upgrade/olm_upgrade_test.go deleted file mode 100644 index 8fb4e65f..00000000 --- a/test/upgrade/olm_upgrade_test.go +++ /dev/null @@ -1,216 +0,0 @@ -/* -Copyright 2023. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "testing" - - . "github.com/onsi/gomega" - . "github.com/project-codeflare/codeflare-common/support" - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - . "github.com/project-codeflare/codeflare-operator/test/e2e" -) - -var ( - namespaceName = "test-ns-olmupgrade" - appWrapperName = "mnist" - jobName = "mnist-job" -) - -func TestMNISTCreateAppWrapper(t *testing.T) { - test := With(t) - test.T().Parallel() - - // Create a namespace - namespace := CreateTestNamespaceWithName(test, namespaceName) - - // Delete namespace only if test failed - defer func() { - if t.Failed() { - DeleteTestNamespace(test, namespace) - } else { - StoreNamespaceLogs(test, namespace) - } - }() - - // Test configuration - config := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "ConfigMap", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "mnist-mcad", - Namespace: namespace.Name, - }, - BinaryData: map[string][]byte{ - // pip requirements - "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), - // MNIST training script - "mnist.py": ReadFile(test, "mnist.py"), - }, - Immutable: Ptr(true), - } - config, err := test.Client().Core().CoreV1().ConfigMaps(namespace.Name).Create(test.Ctx(), config, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created ConfigMap %s/%s successfully", config.Namespace, config.Name) - - // Batch Job - job := &batchv1.Job{ - TypeMeta: metav1.TypeMeta{ - APIVersion: batchv1.SchemeGroupVersion.String(), - Kind: "Job", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: jobName, - Namespace: namespace.Name, - }, - Spec: batchv1.JobSpec{ - Completions: Ptr(int32(1)), - Parallelism: Ptr(int32(1)), - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "job", - Image: GetPyTorchImage(), - Env: []corev1.EnvVar{ - {Name: "PYTHONUSERBASE", Value: "/workdir"}, - {Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()}, - }, - Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"}, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test", - MountPath: "/test", - }, - { - Name: "workdir", - MountPath: "/workdir", - }, - }, - WorkingDir: "/workdir", - }, - }, - Volumes: []corev1.Volume{ - { - Name: "test", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: config.Name, - }, - }, - }, - }, - { - Name: "workdir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - }, - Suspend: Ptr(true), - }, - } - - // Create an AppWrapper resource - aw := &mcadv1beta1.AppWrapper{ - ObjectMeta: metav1.ObjectMeta{ - Name: appWrapperName, - Namespace: namespace.Name, - }, - Spec: mcadv1beta1.AppWrapperSpec{ - AggrResources: mcadv1beta1.AppWrapperResourceList{ - GenericItems: []mcadv1beta1.AppWrapperGenericResource{ - { - DesiredAvailable: 1, - CustomPodResources: []mcadv1beta1.CustomPodResourceTemplate{ - { - Replicas: 1, - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("1G"), - }, - }, - }, - GenericTemplate: Raw(test, job), - CompletionStatus: "Complete", - }, - }, - }, - }, - } - - _, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - test.T().Logf("Created MCAD AppWrapper %s/%s successfully", aw.Namespace, aw.Name) - - test.T().Logf("Waiting for MCAD %s/%s to be running", aw.Namespace, aw.Name) - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). - Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - -} - -func TestMNISTCheckAppWrapperStatus(t *testing.T) { - test := With(t) - test.T().Parallel() - - // get namespace - namespace := GetNamespaceWithName(test, namespaceName) - - //delete the namespace after test complete - defer DeleteTestNamespace(test, namespace) - - // Patch job to resume execution - patch := []byte(`[{"op":"replace","path":"/spec/suspend","value": false}]`) - job, err := test.Client().Core().BatchV1().Jobs(namespace.Name).Patch(test.Ctx(), jobName, types.JSONPatchType, patch, metav1.PatchOptions{}) - test.Expect(err).NotTo(HaveOccurred()) - - test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name) - test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should( - Or( - WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)), - WithTransform(ConditionStatus(batchv1.JobFailed), Equal(corev1.ConditionTrue)), - )) - - test.T().Logf("Waiting for AppWrapper %s/%s to complete", namespace.Name, appWrapperName) - test.Eventually(AppWrapper(test, namespace, appWrapperName), TestTimeoutShort).Should( - Or( - WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)), - WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateFailed)), - )) - - // Assert the AppWrapper has completed successfully - test.Expect(GetAppWrapper(test, namespace, appWrapperName)). - To(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - -} From faff28ab80941db913f509ffdc22eb9cffbed7c8 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account <138894154+codeflare-machine-account@users.noreply.github.com> Date: Fri, 5 Apr 2024 20:01:04 +0200 Subject: [PATCH 317/377] Update dependency versions for release v1.3.0 (#502) Co-authored-by: Bobbins228 --- README.md | 6 +++--- config/manager/params.env | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4f9cd024..0fd60882 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.2.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.2.0) | +| CodeFlare Operator | [v1.3.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.3.0) | | Multi-Cluster App Dispatcher | [v1.40.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.40.0) | -| CodeFlare-SDK | [v0.14.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.14.0) | +| CodeFlare-SDK | [v0.15.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.15.0) | | InstaScale | [v0.4.0](https://github.com/project-codeflare/instascale/releases/tag/v0.4.0) | -| KubeRay | [v1.0.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0) | +| KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | ## Development diff --git a/config/manager/params.env b/config/manager/params.env index c4c8b45f..120dba66 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1 +1 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.2.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.3.0 From 67122ad788019518cfba2d7e546104616756a3c1 Mon Sep 17 00:00:00 2001 From: Wen Zhou Date: Mon, 8 Apr 2024 12:45:46 +0200 Subject: [PATCH 318/377] fix(manifests): remove emptry CRD and reference (#504) --- config/crd/kustomization.yaml | 6 ------ config/default/kustomization.yaml | 1 - 2 files changed, 7 deletions(-) delete mode 100644 config/crd/kustomization.yaml diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml deleted file mode 100644 index 3b60faba..00000000 --- a/config/crd/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# This kustomization.yaml is not intended to be run by itself, -# since it depends on service name and namespace that are out of this kustomize package. -# It should be run by config/default -resources: - -#+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 870c75a5..6e926aed 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -14,7 +14,6 @@ commonLabels: app.kubernetes.io/part-of: codeflare bases: -- ../crd - ../rbac - ../manager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. From ac0ddb5cf552a3f858966d45caa2e0af025c8429 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 8 Apr 2024 16:31:53 +0200 Subject: [PATCH 319/377] Fix OLM upgrade test --- .github/workflows/olm_tests.yaml | 44 +++----------------------------- 1 file changed, 4 insertions(+), 40 deletions(-) diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index b6932d89..472eda20 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -92,10 +92,6 @@ jobs: echo Waiting for Deployment to be ready timeout 60 bash -c 'until [[ $(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}') ]]; do sleep 5 && echo "$(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}')"; done' kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} deployment/codeflare-operator-manager --for=condition=Available=true --timeout=60s - - echo Patch the CodeFlare operator ConfigMap - kubectl patch -n '${{ env.SUBSCRIPTION_NAMESPACE }}' cm codeflare-operator-config --type merge -p '{"data":{"config.yaml":"mcad:\n enabled: true"}}' - env: CATALOG_SOURCE_NAME: "codeflare-olm-test" CATALOG_SOURCE_NAMESPACE: "olm" @@ -107,10 +103,6 @@ jobs: CSV_VERSION=$(kubectl get ClusterServiceVersion -l operators.coreos.com/codeflare-operator.openshift-operators='' -n openshift-operators -o json | jq -r .items[].spec.version) echo "PREVIOUS_VERSION=v$CSV_VERSION" >> $GITHUB_ENV - - name: Deploy CodeFlare stack - run: | - make setup-e2e - - name: Build operator and catalog image run: | make image-push @@ -126,13 +118,6 @@ jobs: BUNDLE_PUSH_OPT: "--tls-verify=false" CATALOG_PUSH_OPT: "--tls-verify=false" - - name: Run OLM Upgrade e2e AppWrapper creation test - run: | - export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} - echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV - set -euo pipefail - go test -timeout 30m -v ./test/upgrade -run TestMNISTCreateAppWrapper -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - - name: Update Operator to the built version run: | ORIGINAL_POD_NAME=$(kubectl get pod -l app.kubernetes.io/name=codeflare-operator -n openshift-operators -o json | jq -r .items[].metadata.name) @@ -162,38 +147,18 @@ jobs: SUBSCRIPTION_NAME: "codeflare-operator" SUBSCRIPTION_NAMESPACE: "openshift-operators" - - name: Run OLM Upgrade e2e Appwrapper Job status test to monitor training - run: | - export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} - echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV - set -euo pipefail - go test -timeout 30m -v ./test/upgrade -run TestMNISTCheckAppWrapperStatus -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - - - name: Run e2e tests against built operator - run: | - export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} - echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV - - set -euo pipefail - go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - - name: Print CodeFlare operator logs if: always() && steps.deploy.outcome == 'success' run: | echo "Printing CodeFlare operator logs" - kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log - - - name: Print KubeRay operator logs - if: always() && steps.deploy.outcome == 'success' - run: | - echo "Printing KubeRay operator logs" - kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log + mkdir logs + kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee logs/codeflare-operator.log - name: Export all KinD pod logs uses: ./common/github-actions/kind-export-logs if: always() && steps.deploy.outcome == 'success' with: - output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} + output-directory: logs - name: Upload logs uses: actions/upload-artifact@v4 @@ -201,5 +166,4 @@ jobs: with: name: logs retention-days: 10 - path: | - ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log + path: logs/**/*.log From 5cc6fd962421184251972da137bfebdfd80eb2d3 Mon Sep 17 00:00:00 2001 From: Shilpa Chugh Date: Fri, 12 Apr 2024 13:23:57 +0530 Subject: [PATCH 320/377] Update kuberay version to latest --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e1b77a8e..fa92d560 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v1.0.0 +KUBERAY_VERSION ?= v1.1.0 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 From 58b23ee3517fa029d3e81012396ca376ec8d4fa5 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 11 Apr 2024 16:09:41 +0200 Subject: [PATCH 321/377] Add missing roles to create ClusterRoleBinding for CodeFlare operator Cleanup after MCAD removal --- ...eflare-operator.clusterserviceversion.yaml | 18 +- config/rbac/admin_role.yaml | 22 -- config/rbac/editor_role.yaml | 24 -- config/rbac/instascale_role.yaml | 43 ---- config/rbac/instascale_role_binding.yaml | 12 - config/rbac/kustomization.yaml | 8 - .../rbac/mcad-controller-ray-clusterrole.yaml | 43 ---- ...cad-controller-ray-clusterrolebinding.yaml | 12 - config/rbac/mcad_manager_role.yaml | 223 ------------------ config/rbac/mcad_manager_role_binding.yaml | 12 - config/rbac/role.yaml | 31 +++ pkg/controllers/raycluster_controller.go | 11 +- 12 files changed, 39 insertions(+), 420 deletions(-) delete mode 100644 config/rbac/admin_role.yaml delete mode 100644 config/rbac/editor_role.yaml delete mode 100644 config/rbac/instascale_role.yaml delete mode 100644 config/rbac/instascale_role_binding.yaml delete mode 100644 config/rbac/mcad-controller-ray-clusterrole.yaml delete mode 100644 config/rbac/mcad-controller-ray-clusterrolebinding.yaml delete mode 100644 config/rbac/mcad_manager_role.yaml delete mode 100644 config/rbac/mcad_manager_role_binding.yaml diff --git a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml index 5531e61a..04a0b301 100644 --- a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml @@ -50,23 +50,7 @@ metadata: namespace: placeholder spec: apiservicedefinitions: {} - customresourcedefinitions: - owned: - - description: AppWrapper is the Schema for the AppWrapper API - displayName: AppWrapper - kind: AppWrappers - name: appwrappers.workload.codeflare.dev - version: v1beta1 - - description: SchedulingSpec is the Schema for the SchedulingSpec API - displayName: SchedulingSpec - kind: SchedulingSpecs - name: schedulingspecs.workload.codeflare.dev - version: v1beta1 - - description: QuotaSubtree is the Schema for the QuotaSubtree API - displayName: QuotaSubtree - kind: QuotaSubtrees - name: quotasubtrees.quota.codeflare.dev - version: v1 + customresourcedefinitions: {} description: CodeFlare allows you to scale complex pipelines anywhere displayName: CodeFlare Operator icon: diff --git a/config/rbac/admin_role.yaml b/config/rbac/admin_role.yaml deleted file mode 100644 index b4f4728c..00000000 --- a/config/rbac/admin_role.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: clusterrole-admin - labels: - rbac.authorization.kubernetes.io/aggregate-to-admin: "true" -rules: -- apiGroups: - - quota.codeflare.dev - resources: - - quotasubtrees - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch diff --git a/config/rbac/editor_role.yaml b/config/rbac/editor_role.yaml deleted file mode 100644 index aa341297..00000000 --- a/config/rbac/editor_role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: clusterrole-edit - labels: - rbac.authorization.k8s.io/aggregate-to-admin: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" -rules: -- apiGroups: - - workload.codeflare.dev - resources: - - schedulingspecs - - appwrappers - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch diff --git a/config/rbac/instascale_role.yaml b/config/rbac/instascale_role.yaml deleted file mode 100644 index dec6b720..00000000 --- a/config/rbac/instascale_role.yaml +++ /dev/null @@ -1,43 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: instascale-role -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - patch - - update -- apiGroups: - - "" - resources: - - secrets - verbs: - - get -- apiGroups: - - config.openshift.io - resources: - - clusterversions - verbs: - - get - - list - - watch -- apiGroups: - - machine.openshift.io - resources: - - machines - - machinesets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch diff --git a/config/rbac/instascale_role_binding.yaml b/config/rbac/instascale_role_binding.yaml deleted file mode 100644 index 00a7d43f..00000000 --- a/config/rbac/instascale_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: instascale-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: instascale-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 12d4bb24..166fe798 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -4,16 +4,8 @@ resources: # if your manager will use a service account that exists at # runtime. Be sure to update RoleBinding and ClusterRoleBinding # subjects if changing service account names. -- admin_role.yaml -- editor_role.yaml - service_account.yaml -- mcad_manager_role.yaml -- mcad_manager_role_binding.yaml - role.yaml - role_binding.yaml -- instascale_role.yaml -- instascale_role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -- mcad-controller-ray-clusterrolebinding.yaml -- mcad-controller-ray-clusterrole.yaml diff --git a/config/rbac/mcad-controller-ray-clusterrole.yaml b/config/rbac/mcad-controller-ray-clusterrole.yaml deleted file mode 100644 index 18e3d98f..00000000 --- a/config/rbac/mcad-controller-ray-clusterrole.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: mcad-controller-ray-clusterrole -rules: -- apiGroups: - - ray.io - resources: - - rayclusters - - rayclusters/finalizers - - rayclusters/status - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - route.openshift.io - resources: - - routes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch diff --git a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml b/config/rbac/mcad-controller-ray-clusterrolebinding.yaml deleted file mode 100644 index da9e8c02..00000000 --- a/config/rbac/mcad-controller-ray-clusterrolebinding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: mcad-controller-ray-clusterrolebinding -subjects: - - kind: ServiceAccount - name: controller-manager - namespace: system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: mcad-controller-ray-clusterrole diff --git a/config/rbac/mcad_manager_role.yaml b/config/rbac/mcad_manager_role.yaml deleted file mode 100644 index b414b8e2..00000000 --- a/config/rbac/mcad_manager_role.yaml +++ /dev/null @@ -1,223 +0,0 @@ ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: manual-manager-role -rules: -- apiGroups: - - '*' - resources: - - deployments - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - deployments - - replicasets - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - config.openshift.io - resources: - - clusterversions - verbs: - - get - - list -- apiGroups: - - coordination.k8s.io - resources: - - kube-scheduler - - leases - verbs: - - create - - get - - update -- apiGroups: - - "" - resources: - - bindings - - pods/binding - verbs: - - create -- apiGroups: - - "" - resources: - - configmaps - - nodes - - persistentvolumeclaims - - persistentvolumes - - secrets - - serviceaccounts - - services - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - endpoints - - kube-scheduler - verbs: - - create - - get - - update -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update -- apiGroups: - - "" - resources: - - kube-scheduler - verbs: - - get - - update -- apiGroups: - - "" - resources: - - pods - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch -- apiGroups: - - "" - resources: - - pods/status - verbs: - - patch - - update -- apiGroups: - - "" - resources: - - replicationcontrollers - verbs: - - get - - list - - watch -- apiGroups: - - events.k8s.io - resources: - - events - - kube-scheduler - verbs: - - create - - patch - - update -- apiGroups: - - machine.openshift.io - resources: - - '*' - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - scheduling.sigs.k8s.io - resources: - - podgroups - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch -- apiGroups: - - storage.k8s.io - resources: - - csidrivers - - csinodes - - csistoragecapacities - verbs: - - get - - list - - watch -- apiGroups: - - workload.codeflare.dev - resources: - - appwrappers - - appwrappers/finalizers - - appwrappers/status - - schedulingspecs - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch -- apiGroups: - - quota.codeflare.dev - resources: - - quotasubtrees - verbs: - - create - - delete - - deletecollection - - get - - list - - patch - - update - - watch diff --git a/config/rbac/mcad_manager_role_binding.yaml b/config/rbac/mcad_manager_role_binding.yaml deleted file mode 100644 index af6c74ae..00000000 --- a/config/rbac/mcad_manager_role_binding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: manual-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: manual-manager-role -subjects: -- kind: ServiceAccount - name: controller-manager - namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e5a2ca9f..70a6a861 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,6 +5,18 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: - "" resources: @@ -19,17 +31,31 @@ rules: resources: - serviceaccounts verbs: + - create - delete - get - patch + - update - apiGroups: - "" resources: - services verbs: + - create + - delete + - get + - patch + - update +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create - delete - get - patch + - update - apiGroups: - ray.io resources: @@ -61,14 +87,19 @@ rules: resources: - clusterrolebindings verbs: + - create - delete - get - patch + - update - apiGroups: - route.openshift.io resources: - routes + - routes/custom-host verbs: + - create - delete - get - patch + - update diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 23104c2a..a098f9b2 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -73,11 +73,14 @@ var ( // +kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=ray.io,resources=rayclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=ray.io,resources=rayclusters/finalizers,verbs=update -// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=patch;delete;get +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes;routes/custom-host,verbs=get;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;create;patch;delete;get -// +kubebuilder:rbac:groups=core,resources=services,verbs=patch;delete;get -// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=patch;delete;get -// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=patch;delete;get +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;create;update;patch;delete +// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create; +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create; // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. From 61d7a48861afa2ccf20b4196e45c26dfc7c50b9f Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 11 Apr 2024 18:41:10 +0100 Subject: [PATCH 322/377] RC Controller - Avoid multiple calls to DiscoveryAPI --- main.go | 6 +++--- pkg/controllers/raycluster_controller.go | 25 ++++++++++++++---------- pkg/controllers/support.go | 25 +++++++++++++++--------- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/main.go b/main.go index 752f146f..8a129e80 100644 --- a/main.go +++ b/main.go @@ -146,9 +146,9 @@ func main() { }) exitOnError(err, "unable to start manager") - v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) - if v { - rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg} + ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) + if ok { + rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg.KubeRay} exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { exitOnError(err, "Could not determine if RayCluster CR present on cluster.") diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index a098f9b2..769a5e09 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -48,11 +48,13 @@ import ( // RayClusterReconciler reconciles a RayCluster object type RayClusterReconciler struct { client.Client - kubeClient *kubernetes.Clientset - routeClient *routev1client.RouteV1Client - Scheme *runtime.Scheme - CookieSalt string - Config *config.CodeFlareOperatorConfiguration + kubeClient *kubernetes.Clientset + routeClient *routev1client.RouteV1Client + Scheme *runtime.Scheme + CookieSalt string + Config *config.KubeRayConfiguration + IsOpenShift bool + IsOpenShiftInitialized bool } const ( @@ -105,8 +107,10 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } isLocalInteractive := annotationBoolVal(ctx, &cluster, "sdk.codeflare.dev/local_interactive", false) - ingressDomain := "" // FIX - CFO will retrieve it. - isOpenShift, ingressHost := getClusterType(ctx, r.kubeClient, &cluster, ingressDomain) + if !r.IsOpenShiftInitialized { + r.IsOpenShift = isOpenShift(ctx, r.kubeClient, &cluster) + r.IsOpenShiftInitialized = true + } if cluster.ObjectMeta.DeletionTimestamp.IsZero() { if !controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { @@ -141,7 +145,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - if cluster.Status.State != "suspended" && r.isRayDashboardOAuthEnabled() && isOpenShift { + if cluster.Status.State != "suspended" && r.isRayDashboardOAuthEnabled() && r.IsOpenShift { logger.Info("Creating OAuth Objects") _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { @@ -182,9 +186,10 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } - } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !isOpenShift { + } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !r.IsOpenShift { + ingressDomain := "" logger.Info("Creating Dashboard Ingress") - _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, ingressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, getIngressHost(ctx, r.kubeClient, &cluster, ingressDomain)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { // This log is info level since errors are not fatal and are expected logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true) diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 348cd03d..44b21bda 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -126,42 +126,49 @@ func getDiscoveryClient(config *rest.Config) (*discovery.DiscoveryClient, error) // Check where we are running. We are trying to distinguish here whether // this is vanilla kubernetes cluster or Openshift -func getClusterType(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressDomain string) (bool, string) { +func isOpenShift(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster) bool { // The discovery package is used to discover APIs supported by a Kubernetes API server. logger := ctrl.LoggerFrom(ctx) config, err := ctrl.GetConfig() if err != nil && config == nil { logger.Info("Cannot retrieve config, assuming we're on Vanilla Kubernetes") - return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) + return false } dclient, err := getDiscoveryClient(config) if err != nil && dclient == nil { logger.Info("Cannot retrieve a DiscoveryClient, assuming we're on Vanilla Kubernetes") - return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) + return false } apiGroupList, err := dclient.ServerGroups() if err != nil { logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes") - return false, "" + return false } for i := 0; i < len(apiGroupList.Groups); i++ { if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") { logger.Info("We detected being on OpenShift!") - return true, "" + return true } } + logger.Info("We detected being on Vanilla Kubernetes!") + return false +} + +// getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain. +func getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressDomain string) string { + logger := ctrl.LoggerFrom(ctx) onKind, _ := isOnKindCluster(clientset) if onKind && ingressDomain == "" { logger.Info("We detected being on a KinD cluster!") - return false, "kind" + return "kind" } logger.Info("We detected being on Vanilla Kubernetes!") - return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) + return fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) } func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { - if r.Config != nil && r.Config.KubeRay != nil && r.Config.KubeRay.RayDashboardOAuthEnabled != nil { - return *r.Config.KubeRay.RayDashboardOAuthEnabled + if r.Config != nil && r.Config.RayDashboardOAuthEnabled != nil { + return *r.Config.RayDashboardOAuthEnabled } return true } From da57f8c3b29f50322ff9867853dc15706bd6d9d0 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Fri, 12 Apr 2024 11:33:27 +0100 Subject: [PATCH 323/377] Add TODO notifier on ingressDomain --- pkg/controllers/raycluster_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 769a5e09..db13f085 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -187,7 +187,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !r.IsOpenShift { - ingressDomain := "" + ingressDomain := "" // TODO: ingressDomain should be retrieved by the CFO and used here to fix local_interactive. Jira: https://issues.redhat.com/browse/RHOAIENG-5330 logger.Info("Creating Dashboard Ingress") _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, getIngressHost(ctx, r.kubeClient, &cluster, ingressDomain)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { From 75a2d234e0f54ff241eca245fe86d8afebe79f74 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Mon, 15 Apr 2024 16:45:11 +0200 Subject: [PATCH 324/377] Add a ConfigMap for e2e tests --- .github/workflows/e2e_tests.yaml | 1 - config/e2e/config.yaml | 8 ++++++++ config/e2e/kustomization.yaml | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 config/e2e/config.yaml diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 2e8fdbc1..0fc1ec88 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -60,7 +60,6 @@ jobs: run: | echo Deploying CodeFlare operator IMG="${REGISTRY_ADDRESS}"/codeflare-operator - sed -i 's/RayDashboardOAuthEnabled: pointer.Bool(true)/RayDashboardOAuthEnabled: pointer.Bool(false)/' main.go make image-push -e IMG="${IMG}" make deploy -e IMG="${IMG}" -e ENV="e2e" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml new file mode 100644 index 00000000..83320bb0 --- /dev/null +++ b/config/e2e/config.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: codeflare-operator-config +data: + config.yaml: | + kuberay: + rayDashboardOAuthEnabled: false diff --git a/config/e2e/kustomization.yaml b/config/e2e/kustomization.yaml index e06123b2..772370da 100644 --- a/config/e2e/kustomization.yaml +++ b/config/e2e/kustomization.yaml @@ -1,6 +1,7 @@ namespace: openshift-operators bases: +- config.yaml - ../default patches: From 689611140bbe12f2576216472d6975d42da6e87d Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 15 Apr 2024 11:46:43 +0100 Subject: [PATCH 325/377] Rayclient Route to resolve its own ingress domain, and enable local_interactive by default --- pkg/controllers/raycluster_controller.go | 15 ++++++--------- pkg/controllers/support.go | 16 ---------------- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index db13f085..f60845eb 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -106,7 +106,6 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, client.IgnoreNotFound(err) } - isLocalInteractive := annotationBoolVal(ctx, &cluster, "sdk.codeflare.dev/local_interactive", false) if !r.IsOpenShiftInitialized { r.IsOpenShift = isOpenShift(ctx, r.kubeClient, &cluster) r.IsOpenShiftInitialized = true @@ -177,13 +176,11 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{RequeueAfter: requeueTime}, err } - if isLocalInteractive { - logger.Info("Creating RayClient Route") - _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredRayClientRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to update RayClient Route") - return ctrl.Result{RequeueAfter: requeueTime}, err - } + logger.Info("Creating RayClient Route") + _, err = r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredRayClientRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update RayClient Route") + return ctrl.Result{RequeueAfter: requeueTime}, err } } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !r.IsOpenShift { @@ -195,7 +192,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } - if isLocalInteractive && ingressDomain != "" { + if ingressDomain != "" { logger.Info("Creating RayClient Ingress") _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, ingressDomain), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 44b21bda..3749fdf0 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -3,7 +3,6 @@ package controllers import ( "context" "fmt" - "strconv" "strings" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -30,7 +29,6 @@ func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConf return routeapply.Route(rayClientNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithSpec(routeapply.RouteSpec(). - WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace). WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(serviceNameFromCluster(cluster)).WithWeight(100)). WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString("client"))). WithTLS(routeapply.TLSConfig().WithTermination("passthrough")), @@ -172,17 +170,3 @@ func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { } return true } - -func annotationBoolVal(ctx context.Context, cluster *rayv1.RayCluster, annotation string, defaultValue bool) bool { - logger := ctrl.LoggerFrom(ctx) - val, exists := cluster.ObjectMeta.Annotations[annotation] - if !exists || val == "" { - return defaultValue - } - boolVal, err := strconv.ParseBool(val) - if err != nil { - logger.Error(err, "Could not convert annotation value to bool", "annotation", annotation, "value", val) - return defaultValue - } - return boolVal -} From 26be493635fff84b651c1fd1258d12040413a7ca Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 15 Apr 2024 16:09:39 +0100 Subject: [PATCH 326/377] Add ingressDomain to configmap to work with local and workflow e2e tests --- config/e2e/config.yaml | 1 + main.go | 1 + pkg/config/config.go | 2 ++ pkg/controllers/raycluster_controller.go | 18 ++++++------ pkg/controllers/support.go | 36 ++++++++---------------- 5 files changed, 24 insertions(+), 34 deletions(-) diff --git a/config/e2e/config.yaml b/config/e2e/config.yaml index 83320bb0..6b8f55a0 100644 --- a/config/e2e/config.yaml +++ b/config/e2e/config.yaml @@ -6,3 +6,4 @@ data: config.yaml: | kuberay: rayDashboardOAuthEnabled: false + ingressDomain: "kind" diff --git a/main.go b/main.go index 8a129e80..58c6c73c 100644 --- a/main.go +++ b/main.go @@ -115,6 +115,7 @@ func main() { }, KubeRay: &config.KubeRayConfiguration{ RayDashboardOAuthEnabled: pointer.Bool(true), + IngressDomain: "fake.domain", }, } diff --git a/pkg/config/config.go b/pkg/config/config.go index 124bb0eb..c71678f1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -33,6 +33,8 @@ type CodeFlareOperatorConfiguration struct { type KubeRayConfiguration struct { RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"` + + IngressDomain string `json:"ingressDomain,omitempty"` } type ControllerManager struct { diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index f60845eb..9fe6dfeb 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -184,21 +184,21 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !r.IsOpenShift { - ingressDomain := "" // TODO: ingressDomain should be retrieved by the CFO and used here to fix local_interactive. Jira: https://issues.redhat.com/browse/RHOAIENG-5330 + logger.Info("We detected being on Vanilla Kubernetes!") logger.Info("Creating Dashboard Ingress") - _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, getIngressHost(ctx, r.kubeClient, &cluster, ingressDomain)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + dashboardName := dashboardNameFromCluster(&cluster) + _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, r.getIngressHost(ctx, r.kubeClient, &cluster, dashboardName)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { // This log is info level since errors are not fatal and are expected logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } - if ingressDomain != "" { - logger.Info("Creating RayClient Ingress") - _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, ingressDomain), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) - if err != nil { - logger.Error(err, "Failed to update RayClient Ingress") - return ctrl.Result{RequeueAfter: requeueTime}, err - } + logger.Info("Creating RayClient Ingress") + rayClientName := rayClientNameFromCluster(&cluster) + _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, r.getIngressHost(ctx, r.kubeClient, &cluster, rayClientName)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update RayClient Ingress") + return ctrl.Result{RequeueAfter: requeueTime}, err } } diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 3749fdf0..49563945 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -8,7 +8,6 @@ import ( rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" networkingv1 "k8s.io/api/networking/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" v1 "k8s.io/client-go/applyconfigurations/meta/v1" @@ -39,7 +38,7 @@ func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConf } // Create an Ingress object for the RayCluster -func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressDomain string) *networkingv1ac.IngressApplyConfiguration { +func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration { return networkingv1ac.Ingress(rayClientNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithAnnotations(map[string]string{ @@ -55,7 +54,7 @@ func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressDomain string) *n WithSpec(networkingv1ac.IngressSpec(). WithIngressClassName("nginx"). WithRules(networkingv1ac.IngressRule(). - WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace + "." + ingressDomain). + WithHost(ingressHost). WithHTTP(networkingv1ac.HTTPIngressRuleValue(). WithPaths(networkingv1ac.HTTPIngressPath(). WithPath("/"). @@ -85,7 +84,7 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo WithUID(types.UID(cluster.UID))). WithSpec(networkingv1ac.IngressSpec(). WithRules(networkingv1ac.IngressRule(). - WithHost(ingressHost). // KinD hostname or ingressDomain + WithHost(ingressHost). // Full Hostname WithHTTP(networkingv1ac.HTTPIngressRuleValue(). WithPaths(networkingv1ac.HTTPIngressPath(). WithPath("/"). @@ -104,19 +103,6 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo ) } -// isOnKindCluster checks if the current cluster is a KinD cluster. -// It searches for a node with a label commonly used by KinD clusters. -func isOnKindCluster(clientset *kubernetes.Clientset) (bool, error) { - nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ - LabelSelector: "kubernetes.io/hostname=kind-control-plane", - }) - if err != nil { - return false, err - } - // If we find one or more nodes with the label, assume it's a KinD cluster. - return len(nodes.Items) > 0, nil -} - // getDiscoveryClient returns a discovery client for the current reconciler func getDiscoveryClient(config *rest.Config) (*discovery.DiscoveryClient, error) { return discovery.NewDiscoveryClientForConfig(config) @@ -153,15 +139,15 @@ func isOpenShift(ctx context.Context, clientset *kubernetes.Clientset, cluster * } // getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain. -func getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressDomain string) string { - logger := ctrl.LoggerFrom(ctx) - onKind, _ := isOnKindCluster(clientset) - if onKind && ingressDomain == "" { - logger.Info("We detected being on a KinD cluster!") - return "kind" +func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressNameFromCluster string) string { + ingressDomain := "fake.domain" + if r.Config != nil && r.Config.IngressDomain != "" { + ingressDomain = r.Config.IngressDomain } - logger.Info("We detected being on Vanilla Kubernetes!") - return fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain) + if ingressDomain == "kind" { + return ingressDomain + } + return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain) } func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { From 61c7bca73efd9a37cd647fdc2159e437355855ff Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 15 Apr 2024 17:57:55 +0100 Subject: [PATCH 327/377] Return error message if IngressDomain configuration not set --- main.go | 2 +- pkg/config/config.go | 2 +- pkg/controllers/raycluster_controller.go | 12 ++++++++++-- pkg/controllers/support.go | 10 ++++++---- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index 58c6c73c..09ebe259 100644 --- a/main.go +++ b/main.go @@ -115,7 +115,7 @@ func main() { }, KubeRay: &config.KubeRayConfiguration{ RayDashboardOAuthEnabled: pointer.Bool(true), - IngressDomain: "fake.domain", + IngressDomain: "", }, } diff --git a/pkg/config/config.go b/pkg/config/config.go index c71678f1..08e2579b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -34,7 +34,7 @@ type CodeFlareOperatorConfiguration struct { type KubeRayConfiguration struct { RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"` - IngressDomain string `json:"ingressDomain,omitempty"` + IngressDomain string `json:"ingressDomain"` } type ControllerManager struct { diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 9fe6dfeb..4f60545f 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -187,7 +187,11 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Info("We detected being on Vanilla Kubernetes!") logger.Info("Creating Dashboard Ingress") dashboardName := dashboardNameFromCluster(&cluster) - _, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, r.getIngressHost(ctx, r.kubeClient, &cluster, dashboardName)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + dashboardIngressHost, err := r.getIngressHost(ctx, r.kubeClient, &cluster, dashboardName) + if err != nil { + return ctrl.Result{RequeueAfter: requeueTime}, err + } + _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, dashboardIngressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { // This log is info level since errors are not fatal and are expected logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true) @@ -195,7 +199,11 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } logger.Info("Creating RayClient Ingress") rayClientName := rayClientNameFromCluster(&cluster) - _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, r.getIngressHost(ctx, r.kubeClient, &cluster, rayClientName)), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + rayClientIngressHost, err := r.getIngressHost(ctx, r.kubeClient, &cluster, rayClientName) + if err != nil { + return ctrl.Result{RequeueAfter: requeueTime}, err + } + _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, rayClientIngressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update RayClient Ingress") return ctrl.Result{RequeueAfter: requeueTime}, err diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 49563945..7871e66a 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -139,15 +139,17 @@ func isOpenShift(ctx context.Context, clientset *kubernetes.Clientset, cluster * } // getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain. -func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressNameFromCluster string) string { - ingressDomain := "fake.domain" +func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressNameFromCluster string) (string, error) { + ingressDomain := "" if r.Config != nil && r.Config.IngressDomain != "" { ingressDomain = r.Config.IngressDomain + } else { + return "", fmt.Errorf("missing IngressDomain configuration in ConfigMap 'codeflare-operator-config'") } if ingressDomain == "kind" { - return ingressDomain + return ingressDomain, nil } - return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain) + return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain), nil } func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { From bc16f94bf2006335cdce7c10546ac36d3b8b6b70 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 15 Apr 2024 20:50:05 +0200 Subject: [PATCH 328/377] Adjust Kuberay dependency in the Makefile --- .github/workflows/tag-and-build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index 870ac35c..a60dceb7 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -85,6 +85,10 @@ jobs: sed -i -E "s|(.*CodeFlare-SDK.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3|" README.md sed -i -E "s|(.*KubeRay.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kuberay-version }}\2${{ github.event.inputs.kuberay-version }}\3|" README.md + - name: Adjust Kuberay dependency in the code + run: | + sed -i -E "s/(.*KUBERAY_VERSION \?= ).*/\1${{ github.event.inputs.kuberay-version }}/" Makefile + - name: Update image version in params.env run: | VERSION=${{ github.event.inputs.version }} perl -i -pe 's/:(.*)$/:$ENV{"VERSION"}/' config/manager/params.env From 8b0902e1426cc6af01e1bb8eaf780f21d8b3e82d Mon Sep 17 00:00:00 2001 From: sutaakar Date: Mon, 15 Apr 2024 19:13:35 +0000 Subject: [PATCH 329/377] Update dependency versions for release v1.3.1 --- Makefile | 2 +- README.md | 6 +++--- config/manager/params.env | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index fa92d560..e1b77a8e 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v1.1.0 +KUBERAY_VERSION ?= v1.0.0 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 diff --git a/README.md b/README.md index 0fd60882..ef2b44d5 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.3.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.3.0) | +| CodeFlare Operator | [v1.3.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.3.1) | | Multi-Cluster App Dispatcher | [v1.40.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.40.0) | -| CodeFlare-SDK | [v0.15.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.15.0) | +| CodeFlare-SDK | [v0.15.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.15.1) | | InstaScale | [v0.4.0](https://github.com/project-codeflare/instascale/releases/tag/v0.4.0) | -| KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | +| KubeRay | [v1.0.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0) | ## Development diff --git a/config/manager/params.env b/config/manager/params.env index 120dba66..36832f91 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1 +1 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.3.0 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.3.1 From c37c4b232b3ef68ae4cc23baeb4a9bf55b775d60 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 15 Apr 2024 21:08:06 +0200 Subject: [PATCH 330/377] E2E tests: Deploy Kuberay before starting CodeFlare operator --- .github/workflows/e2e_tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 0fc1ec88..30ef1ea9 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -58,15 +58,15 @@ jobs: - name: Deploy CodeFlare stack id: deploy run: | + echo Setting up CodeFlare stack + make setup-e2e + echo Deploying CodeFlare operator IMG="${REGISTRY_ADDRESS}"/codeflare-operator make image-push -e IMG="${IMG}" make deploy -e IMG="${IMG}" -e ENV="e2e" kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager - echo Setting up CodeFlare stack - make setup-e2e - - name: Run e2e tests run: | export CODEFLARE_TEST_TIMEOUT_SHORT=1m From f41244597acbc1dc9e2c274fbf5615857b2387b5 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 15 Apr 2024 23:24:06 +0200 Subject: [PATCH 331/377] Use Dnsmasq to provide DNS for KinD --- pkg/controllers/support.go | 3 -- test/e2e/mnist_rayjob_raycluster_test.go | 45 +++++++++++++++++++++++- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 7871e66a..344e6215 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -146,9 +146,6 @@ func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *ku } else { return "", fmt.Errorf("missing IngressDomain configuration in ConfigMap 'codeflare-operator-config'") } - if ingressDomain == "kind" { - return ingressDomain, nil - } return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain), nil } diff --git a/test/e2e/mnist_rayjob_raycluster_test.go b/test/e2e/mnist_rayjob_raycluster_test.go index 46f37d7a..b3cee102 100644 --- a/test/e2e/mnist_rayjob_raycluster_test.go +++ b/test/e2e/mnist_rayjob_raycluster_test.go @@ -17,6 +17,9 @@ limitations under the License. package e2e import ( + "crypto/tls" + "net/http" + "net/url" "testing" . "github.com/onsi/gomega" @@ -221,7 +224,7 @@ func TestMNISTRayJobRayCluster(t *testing.T) { test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("Created RayJob %s/%s successfully", rayJob.Namespace, rayJob.Name) - rayDashboardURL := ExposeService(test, "ray-dashboard", namespace.Name, "raycluster-head-svc", "dashboard") + rayDashboardURL := getRayDashboardURL(test, rayCluster.Namespace, rayCluster.Name) test.T().Logf("Connecting to Ray cluster at: %s", rayDashboardURL.String()) rayClient := NewRayClusterClient(rayDashboardURL) @@ -241,3 +244,43 @@ func TestMNISTRayJobRayCluster(t *testing.T) { test.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). To(WithTransform(RayJobStatus, Equal(rayv1.JobStatusSucceeded))) } + +func getRayDashboardURL(test Test, namespace, rayClusterName string) url.URL { + dashboardName := "ray-dashboard-" + rayClusterName + + if IsOpenShift(test) { + route := GetRoute(test, namespace, dashboardName) + hostname := route.Status.Ingress[0].Host + + // Wait for expected HTTP code + test.T().Logf("Waiting for Route %s/%s to be available", route.Namespace, route.Name) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + + test.Eventually(func() (int, error) { + resp, err := client.Get("https://" + hostname) + if err != nil { + return -1, err + } + return resp.StatusCode, nil + }, TestTimeoutShort).Should(Not(Equal(503))) + + return url.URL{ + Scheme: "https", + Host: hostname, + } + } + + ingress := GetIngress(test, namespace, dashboardName) + + test.T().Logf("Waiting for Ingress %s/%s to be admitted", ingress.Namespace, ingress.Name) + test.Eventually(Ingress(test, ingress.Namespace, ingress.Name), TestTimeoutShort). + Should(WithTransform(LoadBalancerIngresses, HaveLen(1))) + + return url.URL{ + Scheme: "http", + Host: ingress.Spec.Rules[0].Host, + } +} From 3f1166cd29ca9f53682bd02978a14d3834608933 Mon Sep 17 00:00:00 2001 From: Wen Zhou Date: Tue, 16 Apr 2024 11:10:26 +0200 Subject: [PATCH 332/377] fix(manifests): missing namespace in CM stack-config Signed-off-by: Wen Zhou --- config/manager/params.env | 1 + 1 file changed, 1 insertion(+) diff --git a/config/manager/params.env b/config/manager/params.env index 36832f91..8843b5e3 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1 +1,2 @@ codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.3.1 +namespace=opendatahub From 8b3b27c9ea4989a089de953aa2030cb165393d53 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 17 Apr 2024 09:30:32 +0200 Subject: [PATCH 333/377] Remove owner reference for ClusterRoleBinding to RayCluster --- pkg/controllers/raycluster_controller.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 4f60545f..551367e4 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -232,9 +232,6 @@ func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacapply.Cluste WithAPIGroup("rbac.authorization.k8s.io"). WithKind("ClusterRole"). WithName("system:auth-delegator"), - ). - WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) } From 3fa247f1c0c0f94c1f5e8e4e388c235a65bacc1d Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Tue, 16 Apr 2024 12:13:04 +0100 Subject: [PATCH 334/377] add: mutating webhook --- PROJECT | 4 + config/certmanager/certificate.yaml | 3 + config/certmanager/kustomization.yaml | 5 + config/certmanager/kustomizeconfig.yaml | 16 +++ config/default/kustomization.yaml | 11 +- config/default/manager_webhook_patch.yaml | 23 ++++ config/default/webhookcainjection_patch.yaml | 15 +++ config/webhook/kustomization.yaml | 6 + config/webhook/kustomizeconfig.yaml | 25 ++++ config/webhook/manifests.yaml | 27 ++++ config/webhook/service.yaml | 22 ++++ main.go | 3 + pkg/controllers/raycluster_webhook.go | 122 +++++++++++++++++++ 13 files changed, 279 insertions(+), 3 deletions(-) create mode 100644 config/certmanager/certificate.yaml create mode 100644 config/certmanager/kustomization.yaml create mode 100644 config/certmanager/kustomizeconfig.yaml create mode 100644 config/default/manager_webhook_patch.yaml create mode 100644 config/default/webhookcainjection_patch.yaml create mode 100644 config/webhook/kustomization.yaml create mode 100644 config/webhook/kustomizeconfig.yaml create mode 100644 config/webhook/manifests.yaml create mode 100644 config/webhook/service.yaml create mode 100644 pkg/controllers/raycluster_webhook.go diff --git a/PROJECT b/PROJECT index cb690308..3e821aa0 100644 --- a/PROJECT +++ b/PROJECT @@ -15,5 +15,9 @@ resources: domain: codeflare.dev group: ray kind: RayCluster + path: github.com/project-codeflare/codeflare-operator/pkg/controllers version: v1 + webhooks: + defaulting: true + webhookVersion: v1 version: "3" diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml new file mode 100644 index 00000000..45115348 --- /dev/null +++ b/config/certmanager/certificate.yaml @@ -0,0 +1,3 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 00000000..bebea5a5 --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 00000000..e631f777 --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: cert-manager.io + path: spec/commonName +- kind: Certificate + group: cert-manager.io + path: spec/dnsNames diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 6e926aed..fccc7164 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -14,10 +14,15 @@ commonLabels: app.kubernetes.io/part-of: codeflare bases: -- ../rbac -- ../manager + - ../rbac + - ../manager + - ../webhook # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. # - ../prometheus resources: -- metrics_service.yaml + - metrics_service.yaml + +patches: +- path: manager_webhook_patch.yaml +- path: webhookcainjection_patch.yaml diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 00000000..5d6b541d --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: codeflare-operator-raycluster-webhook-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 00000000..2770f6a9 --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,15 @@ +# This patch add annotation to admission webhook config +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: mutatingwebhookconfiguration + app.kubernetes.io/instance: mutating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: codeflare-operator + app.kubernetes.io/part-of: codeflare-operator + app.kubernetes.io/managed-by: kustomize + name: mutating-webhook-configuration + annotations: + service.beta.openshift.io/inject-cabundle: "true" + service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 00000000..9cf26134 --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 00000000..25e21e3c --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 00000000..faf91696 --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + creationTimestamp: null + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-ray-io-v1-raycluster + failurePolicy: Fail + name: mraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayclusters + sideEffects: None diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 00000000..9caf0911 --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: service + app.kubernetes.io/part-of: codeflare-operator + app.kubernetes.io/instance: webhook-service + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: codeflare-operator + app.kubernetes.io/managed-by: kustomize + name: webhook-service + namespace: openshift-operators + annotations: + service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + app.kubernetes.io/part-of: codeflare + app.kubernetes.io/name: codeflare-operator diff --git a/main.go b/main.go index 09ebe259..40f916a5 100644 --- a/main.go +++ b/main.go @@ -147,6 +147,9 @@ func main() { }) exitOnError(err, "unable to start manager") + rayClusterDefaulter := &controllers.RayClusterDefaulter{} + exitOnError(rayClusterDefaulter.SetupWebhookWithManager(mgr), "error setting up webhook") + ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) if ok { rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg.KubeRay} diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go new file mode 100644 index 00000000..5cabf2fa --- /dev/null +++ b/pkg/controllers/raycluster_webhook.go @@ -0,0 +1,122 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +// log is for logging in this package. +var rayclusterlog = logf.Log.WithName("raycluster-resource") + +func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(&rayv1.RayCluster{}). + WithDefaulter(r). + Complete() +} + +//+kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 + +type RayClusterDefaulter struct{} + +var _ webhook.CustomDefaulter = &RayClusterDefaulter{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { + raycluster := obj.(*rayv1.RayCluster) + + rayclusterlog.Info("default", "name", raycluster.Name) + // Check and add OAuth proxy if it does not exist. + alreadyExists := false + for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { + if container.Name == "oauth-proxy" { + rayclusterlog.Info("OAuth sidecar already exists, no patch needed") + alreadyExists = true + break // exits the for loop + } + } + + if !alreadyExists { + rayclusterlog.Info("Adding OAuth sidecar container") + // definition of the new container + newOAuthSidecar := corev1.Container{ + Name: "oauth-proxy", + Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Ports: []corev1.ContainerPort{ + {ContainerPort: 8443, Name: "oauth-proxy"}, + }, + Args: []string{ + "--https-address=:8443", + "--provider=openshift", + "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", + "--upstream=http://localhost:8265", + "--tls-cert=/etc/tls/private/tls.crt", + "--tls-key=/etc/tls/private/tls.key", + "--cookie-secret=$(COOKIE_SECRET)", + "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", + "--added-label=True", + }, + Env: []corev1.EnvVar{ + { + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: raycluster.Name + "-oauth-config", + }, + Key: "cookie_secret", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "proxy-tls-secret", + MountPath: "/etc/tls/private", + ReadOnly: true, + }, + }, + } + + // Adding the new OAuth sidecar container + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) + + tlsSecretVolume := corev1.Volume{ + Name: "proxy-tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: raycluster.Name + "-proxy-tls-secret", + }, + }, + } + + raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) + + // Ensure the service account is set + if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { + raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" + } + } + return nil +} From 2dde5a99d30fbc7e6ac5b6896486b10e04400139 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Tue, 16 Apr 2024 14:32:42 +0100 Subject: [PATCH 335/377] remove: added label --- pkg/controllers/raycluster_webhook.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 5cabf2fa..63581907 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -75,7 +75,6 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e "--tls-key=/etc/tls/private/tls.key", "--cookie-secret=$(COOKIE_SECRET)", "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", - "--added-label=True", }, Env: []corev1.EnvVar{ { From 4afea61ea3d42b57e3d193e5a2d5537812271e5e Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Tue, 16 Apr 2024 16:31:05 +0100 Subject: [PATCH 336/377] add: address feedback on pr --- config/certmanager/certificate.yaml | 3 --- config/certmanager/kustomization.yaml | 5 ----- config/certmanager/kustomizeconfig.yaml | 16 ---------------- config/default/webhookcainjection_patch.yaml | 10 +--------- config/webhook/service.yaml | 11 ++--------- pkg/controllers/raycluster_webhook.go | 4 +++- 6 files changed, 6 insertions(+), 43 deletions(-) delete mode 100644 config/certmanager/certificate.yaml delete mode 100644 config/certmanager/kustomization.yaml delete mode 100644 config/certmanager/kustomizeconfig.yaml diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml deleted file mode 100644 index 45115348..00000000 --- a/config/certmanager/certificate.yaml +++ /dev/null @@ -1,3 +0,0 @@ -# The following manifests contain a self-signed issuer CR and a certificate CR. -# More document can be found at https://docs.cert-manager.io -# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml deleted file mode 100644 index bebea5a5..00000000 --- a/config/certmanager/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: -- certificate.yaml - -configurations: -- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml deleted file mode 100644 index e631f777..00000000 --- a/config/certmanager/kustomizeconfig.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# This configuration is for teaching kustomize how to update name ref and var substitution -nameReference: -- kind: Issuer - group: cert-manager.io - fieldSpecs: - - kind: Certificate - group: cert-manager.io - path: spec/issuerRef/name - -varReference: -- kind: Certificate - group: cert-manager.io - path: spec/commonName -- kind: Certificate - group: cert-manager.io - path: spec/dnsNames diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml index 2770f6a9..39db8634 100644 --- a/config/default/webhookcainjection_patch.yaml +++ b/config/default/webhookcainjection_patch.yaml @@ -2,14 +2,6 @@ apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: - labels: - app.kubernetes.io/name: mutatingwebhookconfiguration - app.kubernetes.io/instance: mutating-webhook-configuration - app.kubernetes.io/component: webhook - app.kubernetes.io/created-by: codeflare-operator - app.kubernetes.io/part-of: codeflare-operator - app.kubernetes.io/managed-by: kustomize name: mutating-webhook-configuration annotations: - service.beta.openshift.io/inject-cabundle: "true" - service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert + service.beta.openshift.io/inject-cabundle: "true" diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml index 9caf0911..ada0e328 100644 --- a/config/webhook/service.yaml +++ b/config/webhook/service.yaml @@ -1,22 +1,15 @@ apiVersion: v1 kind: Service metadata: - labels: - app.kubernetes.io/name: service - app.kubernetes.io/part-of: codeflare-operator - app.kubernetes.io/instance: webhook-service - app.kubernetes.io/component: webhook - app.kubernetes.io/created-by: codeflare-operator - app.kubernetes.io/managed-by: kustomize name: webhook-service namespace: openshift-operators annotations: - service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert + service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert spec: ports: - port: 443 protocol: TCP targetPort: 9443 selector: - app.kubernetes.io/part-of: codeflare + app.kubernetes.io/part-of: codeflare app.kubernetes.io/name: codeflare-operator diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 63581907..bce53322 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -18,7 +18,9 @@ package controllers import ( "context" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" @@ -32,7 +34,7 @@ var rayclusterlog = logf.Log.WithName("raycluster-resource") func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). - WithDefaulter(r). + WithDefaulter(&RayClusterDefaulter{}). Complete() } From 51057f4238da323909ba25efb961d2261336772a Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Wed, 17 Apr 2024 11:26:03 +0100 Subject: [PATCH 337/377] add: check for isRayDashboardOAuthEnabledWebhook before applying patch --- pkg/controllers/raycluster_webhook.go | 128 ++++++++++++++------------ pkg/controllers/support.go | 7 ++ 2 files changed, 76 insertions(+), 59 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index bce53322..3406cd9b 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -26,6 +26,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/project-codeflare/codeflare-operator/pkg/config" ) // log is for logging in this package. @@ -34,13 +36,19 @@ var rayclusterlog = logf.Log.WithName("raycluster-resource") func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error { return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). - WithDefaulter(&RayClusterDefaulter{}). + WithDefaulter(&RayClusterDefaulter{ + Config: r.Config, + rayDashboardOauthEnabled: r.isRayDashboardOAuthEnabledWebhook(), + }). Complete() } //+kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 -type RayClusterDefaulter struct{} +type RayClusterDefaulter struct { + Config *config.KubeRayConfiguration + rayDashboardOauthEnabled bool +} var _ webhook.CustomDefaulter = &RayClusterDefaulter{} @@ -48,75 +56,77 @@ var _ webhook.CustomDefaulter = &RayClusterDefaulter{} func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { raycluster := obj.(*rayv1.RayCluster) - rayclusterlog.Info("default", "name", raycluster.Name) - // Check and add OAuth proxy if it does not exist. - alreadyExists := false - for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { - if container.Name == "oauth-proxy" { - rayclusterlog.Info("OAuth sidecar already exists, no patch needed") - alreadyExists = true - break // exits the for loop + if r.rayDashboardOauthEnabled { + rayclusterlog.Info("default", "name", raycluster.Name) + // Check and add OAuth proxy if it does not exist. + alreadyExists := false + for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { + if container.Name == "oauth-proxy" { + rayclusterlog.Info("OAuth sidecar already exists, no patch needed") + alreadyExists = true + break // exits the for loop + } } - } - if !alreadyExists { - rayclusterlog.Info("Adding OAuth sidecar container") - // definition of the new container - newOAuthSidecar := corev1.Container{ - Name: "oauth-proxy", - Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", - Ports: []corev1.ContainerPort{ - {ContainerPort: 8443, Name: "oauth-proxy"}, - }, - Args: []string{ - "--https-address=:8443", - "--provider=openshift", - "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", - "--upstream=http://localhost:8265", - "--tls-cert=/etc/tls/private/tls.crt", - "--tls-key=/etc/tls/private/tls.key", - "--cookie-secret=$(COOKIE_SECRET)", - "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", - }, - Env: []corev1.EnvVar{ - { - Name: "COOKIE_SECRET", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: raycluster.Name + "-oauth-config", + if !alreadyExists { + rayclusterlog.Info("Adding OAuth sidecar container") + // definition of the new container + newOAuthSidecar := corev1.Container{ + Name: "oauth-proxy", + Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Ports: []corev1.ContainerPort{ + {ContainerPort: 8443, Name: "oauth-proxy"}, + }, + Args: []string{ + "--https-address=:8443", + "--provider=openshift", + "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", + "--upstream=http://localhost:8265", + "--tls-cert=/etc/tls/private/tls.crt", + "--tls-key=/etc/tls/private/tls.key", + "--cookie-secret=$(COOKIE_SECRET)", + "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", + }, + Env: []corev1.EnvVar{ + { + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: raycluster.Name + "-oauth-config", + }, + Key: "cookie_secret", }, - Key: "cookie_secret", }, }, }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "proxy-tls-secret", - MountPath: "/etc/tls/private", - ReadOnly: true, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "proxy-tls-secret", + MountPath: "/etc/tls/private", + ReadOnly: true, + }, }, - }, - } + } - // Adding the new OAuth sidecar container - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) + // Adding the new OAuth sidecar container + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) - tlsSecretVolume := corev1.Volume{ - Name: "proxy-tls-secret", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: raycluster.Name + "-proxy-tls-secret", + tlsSecretVolume := corev1.Volume{ + Name: "proxy-tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: raycluster.Name + "-proxy-tls-secret", + }, }, - }, - } + } - raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) + raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) - // Ensure the service account is set - if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { - raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" + // Ensure the service account is set + if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { + raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" + } } } return nil diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 344e6215..698bd8da 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -155,3 +155,10 @@ func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { } return true } + +func (r *RayClusterDefaulter) isRayDashboardOAuthEnabledWebhook() bool { + if r.Config != nil && r.Config.RayDashboardOAuthEnabled != nil { + return *r.Config.RayDashboardOAuthEnabled + } + return true +} From 77accff191dd2574e020cf5a4f8043b31a9c0ef1 Mon Sep 17 00:00:00 2001 From: Eoin Gallinagh Date: Wed, 17 Apr 2024 12:47:55 +0100 Subject: [PATCH 338/377] update: placement of cookie secret in the mutating webhook for rayclusters --- pkg/controllers/raycluster_webhook.go | 30 +++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 3406cd9b..d63616e1 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -87,19 +87,6 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e "--cookie-secret=$(COOKIE_SECRET)", "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", }, - Env: []corev1.EnvVar{ - { - Name: "COOKIE_SECRET", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: raycluster.Name + "-oauth-config", - }, - Key: "cookie_secret", - }, - }, - }, - }, VolumeMounts: []corev1.VolumeMount{ { Name: "proxy-tls-secret", @@ -112,6 +99,23 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e // Adding the new OAuth sidecar container raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) + cookieSecret := corev1.EnvVar{ + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: raycluster.Name + "-oauth-config", + }, + Key: "cookie_secret", + }, + }, + } + + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = append( + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, + cookieSecret, + ) + tlsSecretVolume := corev1.Volume{ Name: "proxy-tls-secret", VolumeSource: corev1.VolumeSource{ From 06efef73f1377d1b6898dabcb86bbdeeda09c256 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 17 Apr 2024 16:47:51 +0200 Subject: [PATCH 339/377] Refactor RayCluster mutating webhook --- PROJECT | 2 +- config/default/kustomization.yaml | 5 - config/openshift/kustomization.yaml | 22 +++ .../manager_webhook_patch.yaml | 0 .../webhookcainjection_patch.yaml | 0 main.go | 3 +- pkg/controllers/raycluster_webhook.go | 159 +++++++++--------- pkg/controllers/support.go | 7 - 8 files changed, 102 insertions(+), 96 deletions(-) create mode 100644 config/openshift/kustomization.yaml rename config/{default => openshift}/manager_webhook_patch.yaml (100%) rename config/{default => openshift}/webhookcainjection_patch.yaml (100%) diff --git a/PROJECT b/PROJECT index 3e821aa0..0fc5562e 100644 --- a/PROJECT +++ b/PROJECT @@ -12,7 +12,7 @@ projectName: codeflare-operator repo: github.com/project-codeflare/codeflare-operator resources: - controller: true - domain: codeflare.dev + domain: ray.io group: ray kind: RayCluster path: github.com/project-codeflare/codeflare-operator/pkg/controllers diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index fccc7164..42fa7179 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -16,13 +16,8 @@ commonLabels: bases: - ../rbac - ../manager - - ../webhook # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. # - ../prometheus resources: - metrics_service.yaml - -patches: -- path: manager_webhook_patch.yaml -- path: webhookcainjection_patch.yaml diff --git a/config/openshift/kustomization.yaml b/config/openshift/kustomization.yaml new file mode 100644 index 00000000..22e16816 --- /dev/null +++ b/config/openshift/kustomization.yaml @@ -0,0 +1,22 @@ +# Adds namespace to all resources. +namespace: openshift-operators + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: codeflare-operator- + +# Labels to add to all resources and selectors. +commonLabels: + app.kubernetes.io/name: codeflare-operator + app.kubernetes.io/part-of: codeflare + +bases: + - ../default + - ../webhook + +patches: +- path: manager_webhook_patch.yaml +- path: webhookcainjection_patch.yaml diff --git a/config/default/manager_webhook_patch.yaml b/config/openshift/manager_webhook_patch.yaml similarity index 100% rename from config/default/manager_webhook_patch.yaml rename to config/openshift/manager_webhook_patch.yaml diff --git a/config/default/webhookcainjection_patch.yaml b/config/openshift/webhookcainjection_patch.yaml similarity index 100% rename from config/default/webhookcainjection_patch.yaml rename to config/openshift/webhookcainjection_patch.yaml diff --git a/main.go b/main.go index 40f916a5..1ed191bd 100644 --- a/main.go +++ b/main.go @@ -147,8 +147,7 @@ func main() { }) exitOnError(err, "unable to start manager") - rayClusterDefaulter := &controllers.RayClusterDefaulter{} - exitOnError(rayClusterDefaulter.SetupWebhookWithManager(mgr), "error setting up webhook") + exitOnError(controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay), "error setting up RayCluster webhook") ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) if ok { diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index d63616e1..0cd63504 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -23,6 +23,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -33,105 +34,101 @@ import ( // log is for logging in this package. var rayclusterlog = logf.Log.WithName("raycluster-resource") -func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error { +func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConfiguration) error { return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). - WithDefaulter(&RayClusterDefaulter{ - Config: r.Config, - rayDashboardOauthEnabled: r.isRayDashboardOAuthEnabledWebhook(), + WithDefaulter(&rayClusterDefaulter{ + Config: cfg, }). Complete() } -//+kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 -type RayClusterDefaulter struct { - Config *config.KubeRayConfiguration - rayDashboardOauthEnabled bool +type rayClusterDefaulter struct { + Config *config.KubeRayConfiguration } -var _ webhook.CustomDefaulter = &RayClusterDefaulter{} +var _ webhook.CustomDefaulter = &rayClusterDefaulter{} // Default implements webhook.Defaulter so a webhook will be registered for the type -func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { +func (r *rayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { raycluster := obj.(*rayv1.RayCluster) - if r.rayDashboardOauthEnabled { - rayclusterlog.Info("default", "name", raycluster.Name) - // Check and add OAuth proxy if it does not exist. - alreadyExists := false - for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { - if container.Name == "oauth-proxy" { - rayclusterlog.Info("OAuth sidecar already exists, no patch needed") - alreadyExists = true - break // exits the for loop - } + if !pointer.BoolDeref(r.Config.RayDashboardOAuthEnabled, true) { + return nil + } + + // Check and add OAuth proxy if it does not exist + for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { + if container.Name == "oauth-proxy" { + rayclusterlog.V(2).Info("OAuth sidecar already exists, no patch needed") + return nil } + } - if !alreadyExists { - rayclusterlog.Info("Adding OAuth sidecar container") - // definition of the new container - newOAuthSidecar := corev1.Container{ - Name: "oauth-proxy", - Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", - Ports: []corev1.ContainerPort{ - {ContainerPort: 8443, Name: "oauth-proxy"}, - }, - Args: []string{ - "--https-address=:8443", - "--provider=openshift", - "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", - "--upstream=http://localhost:8265", - "--tls-cert=/etc/tls/private/tls.crt", - "--tls-key=/etc/tls/private/tls.key", - "--cookie-secret=$(COOKIE_SECRET)", - "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "proxy-tls-secret", - MountPath: "/etc/tls/private", - ReadOnly: true, - }, - }, - } - - // Adding the new OAuth sidecar container - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) - - cookieSecret := corev1.EnvVar{ - Name: "COOKIE_SECRET", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: raycluster.Name + "-oauth-config", - }, - Key: "cookie_secret", - }, - }, - } - - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = append( - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, - cookieSecret, - ) - - tlsSecretVolume := corev1.Volume{ - Name: "proxy-tls-secret", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: raycluster.Name + "-proxy-tls-secret", - }, + rayclusterlog.V(2).Info("Adding OAuth sidecar container") + // definition of the new container + newOAuthSidecar := corev1.Container{ + Name: "oauth-proxy", + Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", + Ports: []corev1.ContainerPort{ + {ContainerPort: 8443, Name: "oauth-proxy"}, + }, + Args: []string{ + "--https-address=:8443", + "--provider=openshift", + "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", + "--upstream=http://localhost:8265", + "--tls-cert=/etc/tls/private/tls.crt", + "--tls-key=/etc/tls/private/tls.key", + "--cookie-secret=$(COOKIE_SECRET)", + "--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "proxy-tls-secret", + MountPath: "/etc/tls/private", + ReadOnly: true, + }, + }, + } + + // Adding the new OAuth sidecar container + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) + + cookieSecret := corev1.EnvVar{ + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: raycluster.Name + "-oauth-config", }, - } + Key: "cookie_secret", + }, + }, + } - raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = append( + raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, + cookieSecret, + ) + + tlsSecretVolume := corev1.Volume{ + Name: "proxy-tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: raycluster.Name + "-proxy-tls-secret", + }, + }, + } - // Ensure the service account is set - if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { - raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" - } - } + raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) + + // Ensure the service account is set + if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { + raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" } + return nil } diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 698bd8da..344e6215 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -155,10 +155,3 @@ func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { } return true } - -func (r *RayClusterDefaulter) isRayDashboardOAuthEnabledWebhook() bool { - if r.Config != nil && r.Config.RayDashboardOAuthEnabled != nil { - return *r.Config.RayDashboardOAuthEnabled - } - return true -} From 28a9c28d69182d4f62f7ce13aa35ca7eced26f22 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 17 Apr 2024 17:30:54 +0200 Subject: [PATCH 340/377] Start RayCluster webhook on OpenShift only --- main.go | 35 ++++++++++++++++++--- pkg/controllers/raycluster_controller.go | 18 ++++------- pkg/controllers/support.go | 39 ------------------------ 3 files changed, 37 insertions(+), 55 deletions(-) diff --git a/main.go b/main.go index 1ed191bd..7eff5fc4 100644 --- a/main.go +++ b/main.go @@ -147,11 +147,21 @@ func main() { }) exitOnError(err, "unable to start manager") - exitOnError(controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay), "error setting up RayCluster webhook") + OpenShift := isOpenShift(ctx, kubeClient.DiscoveryClient) - ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) + if OpenShift { + // TODO: setup the RayCluster webhook on vanilla Kubernetes + exitOnError(controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay), "error setting up RayCluster webhook") + } + + ok, err := hasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) if ok { - rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg.KubeRay} + rayClusterController := controllers.RayClusterReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Config: cfg.KubeRay, + IsOpenShift: OpenShift, + } exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { exitOnError(err, "Could not determine if RayCluster CR present on cluster.") @@ -207,7 +217,7 @@ func createConfigMap(ctx context.Context, client kubernetes.Interface, ns, name return err } -func HasAPIResourceForGVK(dc discovery.DiscoveryInterface, gvk schema.GroupVersionKind) (bool, error) { +func hasAPIResourceForGVK(dc discovery.DiscoveryInterface, gvk schema.GroupVersionKind) (bool, error) { gv, kind := gvk.ToAPIVersionAndKind() if resources, err := dc.ServerResourcesForGroupVersion(gv); err != nil { if apierrors.IsNotFound(err) { @@ -247,3 +257,20 @@ func exitOnError(err error, msg string) { os.Exit(1) } } + +func isOpenShift(ctx context.Context, dc discovery.DiscoveryInterface) bool { + logger := ctrl.LoggerFrom(ctx) + apiGroupList, err := dc.ServerGroups() + if err != nil { + logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes") + return false + } + for i := 0; i < len(apiGroupList.Groups); i++ { + if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") { + logger.Info("We detected being on OpenShift!") + return true + } + } + logger.Info("We detected being on Vanilla Kubernetes!") + return false +} diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 551367e4..461e3d01 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -48,13 +48,12 @@ import ( // RayClusterReconciler reconciles a RayCluster object type RayClusterReconciler struct { client.Client - kubeClient *kubernetes.Clientset - routeClient *routev1client.RouteV1Client - Scheme *runtime.Scheme - CookieSalt string - Config *config.KubeRayConfiguration - IsOpenShift bool - IsOpenShiftInitialized bool + kubeClient *kubernetes.Clientset + routeClient *routev1client.RouteV1Client + Scheme *runtime.Scheme + CookieSalt string + Config *config.KubeRayConfiguration + IsOpenShift bool } const ( @@ -106,11 +105,6 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, client.IgnoreNotFound(err) } - if !r.IsOpenShiftInitialized { - r.IsOpenShift = isOpenShift(ctx, r.kubeClient, &cluster) - r.IsOpenShiftInitialized = true - } - if cluster.ObjectMeta.DeletionTimestamp.IsZero() { if !controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { logger.Info("Add a finalizer", "finalizer", oAuthFinalizer) diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 344e6215..5a47993b 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -3,7 +3,6 @@ package controllers import ( "context" "fmt" - "strings" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -12,10 +11,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" v1 "k8s.io/client-go/applyconfigurations/meta/v1" networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" - "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" ) @@ -103,41 +99,6 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo ) } -// getDiscoveryClient returns a discovery client for the current reconciler -func getDiscoveryClient(config *rest.Config) (*discovery.DiscoveryClient, error) { - return discovery.NewDiscoveryClientForConfig(config) -} - -// Check where we are running. We are trying to distinguish here whether -// this is vanilla kubernetes cluster or Openshift -func isOpenShift(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster) bool { - // The discovery package is used to discover APIs supported by a Kubernetes API server. - logger := ctrl.LoggerFrom(ctx) - config, err := ctrl.GetConfig() - if err != nil && config == nil { - logger.Info("Cannot retrieve config, assuming we're on Vanilla Kubernetes") - return false - } - dclient, err := getDiscoveryClient(config) - if err != nil && dclient == nil { - logger.Info("Cannot retrieve a DiscoveryClient, assuming we're on Vanilla Kubernetes") - return false - } - apiGroupList, err := dclient.ServerGroups() - if err != nil { - logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes") - return false - } - for i := 0; i < len(apiGroupList.Groups); i++ { - if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") { - logger.Info("We detected being on OpenShift!") - return true - } - } - logger.Info("We detected being on Vanilla Kubernetes!") - return false -} - // getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain. func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressNameFromCluster string) (string, error) { ingressDomain := "" From 4a7cb60bce2381e6b735af87ab6fb1d784b41606 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Wed, 17 Apr 2024 17:58:25 +0200 Subject: [PATCH 341/377] Refactor RayCluster controller --- pkg/controllers/raycluster_controller.go | 65 +++++++++++++++--------- pkg/controllers/support.go | 22 -------- 2 files changed, 42 insertions(+), 45 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 461e3d01..ba6ca84f 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -21,6 +21,7 @@ import ( "crypto/rand" "crypto/sha1" "encoding/base64" + "fmt" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -96,9 +97,9 @@ var ( func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := ctrl.LoggerFrom(ctx) - var cluster rayv1.RayCluster + cluster := &rayv1.RayCluster{} - if err := r.Get(ctx, req.NamespacedName, &cluster); err != nil { + if err := r.Get(ctx, req.NamespacedName, cluster); err != nil { if !errors.IsNotFound(err) { logger.Error(err, "Error getting RayCluster resource") } @@ -106,21 +107,21 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if cluster.ObjectMeta.DeletionTimestamp.IsZero() { - if !controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { + if !controllerutil.ContainsFinalizer(cluster, oAuthFinalizer) { logger.Info("Add a finalizer", "finalizer", oAuthFinalizer) - controllerutil.AddFinalizer(&cluster, oAuthFinalizer) - if err := r.Update(ctx, &cluster); err != nil { + controllerutil.AddFinalizer(cluster, oAuthFinalizer) + if err := r.Update(ctx, cluster); err != nil { // this log is info level since errors are not fatal and are expected logger.Info("WARN: Failed to update RayCluster with finalizer", "error", err.Error(), logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } } - } else if controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) { + } else if controllerutil.ContainsFinalizer(cluster, oAuthFinalizer) { err := client.IgnoreNotFound(r.Client.Delete( ctx, &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: crbNameFromCluster(&cluster), + Name: crbNameFromCluster(cluster), }, }, &deleteOptions, @@ -129,8 +130,8 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Error(err, "Failed to remove OAuth ClusterRoleBinding.", logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } - controllerutil.RemoveFinalizer(&cluster, oAuthFinalizer) - if err := r.Update(ctx, &cluster); err != nil { + controllerutil.RemoveFinalizer(cluster, oAuthFinalizer) + if err := r.Update(ctx, cluster); err != nil { logger.Error(err, "Failed to remove finalizer from RayCluster", logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } @@ -138,66 +139,66 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - if cluster.Status.State != "suspended" && r.isRayDashboardOAuthEnabled() && r.IsOpenShift { + if cluster.Status.State != "suspended" && isRayDashboardOAuthEnabled(r.Config) && r.IsOpenShift { logger.Info("Creating OAuth Objects") - _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update OAuth Route") return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(&cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to create OAuth Secret") return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.CoreV1().Services(cluster.Namespace).Apply(ctx, desiredOAuthService(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.CoreV1().Services(cluster.Namespace).Apply(ctx, desiredOAuthService(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update OAuth Service") return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Apply(ctx, desiredServiceAccount(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Apply(ctx, desiredServiceAccount(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update OAuth ServiceAccount") return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update OAuth ClusterRoleBinding") return ctrl.Result{RequeueAfter: requeueTime}, err } logger.Info("Creating RayClient Route") - _, err = r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredRayClientRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredRayClientRoute(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update RayClient Route") return ctrl.Result{RequeueAfter: requeueTime}, err } - } else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !r.IsOpenShift { + } else if cluster.Status.State != "suspended" && !isRayDashboardOAuthEnabled(r.Config) && !r.IsOpenShift { logger.Info("We detected being on Vanilla Kubernetes!") logger.Info("Creating Dashboard Ingress") - dashboardName := dashboardNameFromCluster(&cluster) - dashboardIngressHost, err := r.getIngressHost(ctx, r.kubeClient, &cluster, dashboardName) + dashboardName := dashboardNameFromCluster(cluster) + dashboardIngressHost, err := getIngressHost(r.Config, cluster, dashboardName) if err != nil { return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, dashboardIngressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(cluster, dashboardIngressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { // This log is info level since errors are not fatal and are expected logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true) return ctrl.Result{RequeueAfter: requeueTime}, err } logger.Info("Creating RayClient Ingress") - rayClientName := rayClientNameFromCluster(&cluster) - rayClientIngressHost, err := r.getIngressHost(ctx, r.kubeClient, &cluster, rayClientName) + rayClientName := rayClientNameFromCluster(cluster) + rayClientIngressHost, err := getIngressHost(r.Config, cluster, rayClientName) if err != nil { return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, rayClientIngressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(cluster, rayClientIngressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update RayClient Ingress") return ctrl.Result{RequeueAfter: requeueTime}, err @@ -207,6 +208,24 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } +// getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain. +func getIngressHost(cfg *config.KubeRayConfiguration, cluster *rayv1.RayCluster, ingressNameFromCluster string) (string, error) { + ingressDomain := "" + if cfg != nil && cfg.IngressDomain != "" { + ingressDomain = cfg.IngressDomain + } else { + return "", fmt.Errorf("missing IngressDomain configuration in ConfigMap 'codeflare-operator-config'") + } + return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain), nil +} + +func isRayDashboardOAuthEnabled(cfg *config.KubeRayConfiguration) bool { + if cfg != nil && cfg.RayDashboardOAuthEnabled != nil { + return *cfg.RayDashboardOAuthEnabled + } + return true +} + func crbNameFromCluster(cluster *rayv1.RayCluster) string { return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz} } diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 5a47993b..673d5941 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -1,9 +1,6 @@ package controllers import ( - "context" - "fmt" - rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" networkingv1 "k8s.io/api/networking/v1" @@ -11,7 +8,6 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" v1 "k8s.io/client-go/applyconfigurations/meta/v1" networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" - "k8s.io/client-go/kubernetes" routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" ) @@ -98,21 +94,3 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo ), ) } - -// getIngressHost generates the cluster URL string based on the cluster type, RayCluster, and ingress domain. -func (r *RayClusterReconciler) getIngressHost(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressNameFromCluster string) (string, error) { - ingressDomain := "" - if r.Config != nil && r.Config.IngressDomain != "" { - ingressDomain = r.Config.IngressDomain - } else { - return "", fmt.Errorf("missing IngressDomain configuration in ConfigMap 'codeflare-operator-config'") - } - return fmt.Sprintf("%s-%s.%s", ingressNameFromCluster, cluster.Namespace, ingressDomain), nil -} - -func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool { - if r.Config != nil && r.Config.RayDashboardOAuthEnabled != nil { - return *r.Config.RayDashboardOAuthEnabled - } - return true -} From 9f5bde6b52a2810a7cb6922a2c25100e4641c399 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Fri, 9 Feb 2024 13:47:58 +0100 Subject: [PATCH 342/377] Adjust env tests for RayCluster controller --- .github/workflows/component_tests.yaml | 43 ++++ Makefile | 10 + pkg/controllers/raycluster_controller_test.go | 206 ++++++++---------- pkg/controllers/suite_test.go | 32 ++- 4 files changed, 168 insertions(+), 123 deletions(-) create mode 100644 .github/workflows/component_tests.yaml diff --git a/.github/workflows/component_tests.yaml b/.github/workflows/component_tests.yaml new file mode 100644 index 00000000..815025b2 --- /dev/null +++ b/.github/workflows/component_tests.yaml @@ -0,0 +1,43 @@ +name: Component tests + +on: + pull_request: + branches: + - main + - 'release-*' + paths-ignore: + - 'docs/**' + - '**.adoc' + - '**.md' + - 'LICENSE' + push: + branches: + - main + - 'release-*' + paths-ignore: + - 'docs/**' + - '**.adoc' + - '**.md' + - 'LICENSE' + +concurrency: + group: ${{ github.head_ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + kubernetes-component: + + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set Go + uses: actions/setup-go@v5 + with: + go-version: v1.20 + + - name: Run component tests + run: | + make test-component diff --git a/Makefile b/Makefile index e1b77a8e..cb8cd533 100644 --- a/Makefile +++ b/Makefile @@ -205,6 +205,7 @@ $(LOCALBIN): KUSTOMIZE ?= $(LOCALBIN)/kustomize YQ ?= $(LOCALBIN)/yq CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +GINKGO ?= $(LOCALBIN)/ginkgo ENVTEST ?= $(LOCALBIN)/setup-envtest OPENSHIFT-GOIMPORTS ?= $(LOCALBIN)/openshift-goimports OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk @@ -241,6 +242,11 @@ controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessar $(CONTROLLER_GEN): $(LOCALBIN) test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) +.PHONY: ginkgo +ginkgo: $(GINKGO) ## Download ginkgo locally if necessary. +$(GINKGO): $(LOCALBIN) + test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo + .PHONY: install-yq install-yq: $(YQ) ## Download yq locally if necessary $(YQ): $(LOCALBIN) @@ -350,6 +356,10 @@ catalog-push: ## Push a catalog image. test-unit: manifests fmt vet envtest ## Run unit tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(go list ./... | grep -v /test/) -coverprofile cover.out +.PHONY: test-component +test-component: envtest ginkgo ## Run component tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GINKGO) -v ./pkg/controllers/ + .PHONY: test-e2e test-e2e: manifests fmt vet ## Run e2e tests. go test -timeout 30m -v ./test/e2e diff --git a/pkg/controllers/raycluster_controller_test.go b/pkg/controllers/raycluster_controller_test.go index 6592aa29..94958b33 100644 --- a/pkg/controllers/raycluster_controller_test.go +++ b/pkg/controllers/raycluster_controller_test.go @@ -17,8 +17,6 @@ limitations under the License. package controllers import ( - "context" - "math/rand" "time" . "github.com/onsi/ginkgo/v2" @@ -29,158 +27,144 @@ import ( rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" routev1 "github.com/openshift/api/route/v1" ) -func stringInList(l []string, s string) bool { - for _, i := range l { - if i == s { - return true - } - } - return false -} - -var letters = []rune("abcdefghijklmnopqrstuvwxyz") -var r = rand.New(rand.NewSource(time.Now().UnixNano())) - -func randSeq(n int) string { - b := make([]rune, n) - for i := range b { - b[i] = letters[r.Intn(len(letters))] - } - return string(b) -} - var _ = Describe("RayCluster controller", func() { Context("RayCluster controller test", func() { var rayClusterName = "test-raycluster" - var typeNamespaceName types.NamespacedName - ctx := context.Background() - BeforeEach(func() { - By("Generate random number so each run is creating unique") - rString := randSeq(10) - rayClusterName = rayClusterName + "-" + rString - typeNamespaceName = types.NamespacedName{Name: rayClusterName, Namespace: rayClusterName} + var namespaceName string + BeforeEach(func(ctx SpecContext) { By("Creating a namespace for running the tests.") namespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: rayClusterName, + GenerateName: "test-", }, } - var err error - err = k8sClient.Create(ctx, namespace) - Expect(err).To(Not(HaveOccurred())) + namespace, err := k8sClient.CoreV1().Namespaces().Create(ctx, namespace, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func(ctx SpecContext) { + err := k8sClient.CoreV1().Namespaces().Delete(ctx, namespace.Name, metav1.DeleteOptions{}) + Expect(err).To(Not(HaveOccurred())) + }) + namespaceName = namespace.Name By("creating a basic instance of the RayCluster CR") raycluster := &rayv1.RayCluster{ ObjectMeta: metav1.ObjectMeta{ Name: rayClusterName, - Namespace: rayClusterName, + Namespace: namespace.Name, }, Spec: rayv1.RayClusterSpec{ HeadGroupSpec: rayv1.HeadGroupSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - corev1.Container{}, - }, + Containers: []corev1.Container{}, }, }, RayStartParams: map[string]string{}, }, }, } - err = k8sClient.Get(ctx, typeNamespaceName, &rayv1.RayCluster{}) - Expect(errors.IsNotFound(err)).To(Equal(true)) - err = k8sClient.Create(ctx, raycluster) + _, err = rayClient.RayV1().RayClusters(namespace.Name).Create(ctx, raycluster, metav1.CreateOptions{}) Expect(err).To(Not(HaveOccurred())) }) - AfterEach(func() { - By("removing the instance of the RayCluster used") - // err := clientSet.CoreV1().Namespaces().Delete(ctx, RayClusterName, metav1.DeleteOptions{}) - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - if err != nil { - Expect(errors.IsNotFound(err)).To(Equal(true)) - } else { + AfterEach(func(ctx SpecContext) { + By("removing instances of the RayClusters used") + rayClusters, err := rayClient.RayV1().RayClusters(namespaceName).List(ctx, metav1.ListOptions{}) + Expect(err).To(Not(HaveOccurred())) + + for _, rayCluster := range rayClusters.Items { + err = rayClient.RayV1().RayClusters(namespaceName).Delete(ctx, rayCluster.Name, metav1.DeleteOptions{}) Expect(err).To(Not(HaveOccurred())) - _ = k8sClient.Delete(ctx, &foundRayCluster) } - Eventually(func() bool { - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - return errors.IsNotFound(err) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) - }) - It("should have oauth finalizer set", func() { - foundRayCluster := rayv1.RayCluster{} - Eventually(func() bool { - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - Expect(err).To(Not(HaveOccurred())) - return stringInList(foundRayCluster.Finalizers, oAuthFinalizer) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) + Eventually(func() ([]rayv1.RayCluster, error) { + rayClusters, err := rayClient.RayV1().RayClusters(namespaceName).List(ctx, metav1.ListOptions{}) + return rayClusters.Items, err + }).WithTimeout(time.Second * 10).Should(BeEmpty()) }) - It("should create all oauth resources", func() { - Eventually(func() error { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) - if err != nil { - return err - } - err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) - if err != nil { - return err - } - return nil - }, SpecTimeout(time.Second*10)).Should(Not(HaveOccurred())) - }) + It("should have oauth finalizer set", func(ctx SpecContext) { + Eventually(func() ([]string, error) { + foundRayCluster, err := rayClient.RayV1().RayClusters(namespaceName).Get(ctx, rayClusterName, metav1.GetOptions{}) + return foundRayCluster.Finalizers, err + }).WithTimeout(time.Second * 10).Should(ContainElement(oAuthFinalizer)) + }, SpecTimeout(time.Second*10)) - It("should set owner references for all resources", func() { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) - Expect(err).ToNot(HaveOccurred()) - err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthSecretNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Secret{}) - Expect(err).To(Not(HaveOccurred())) - err = k8sClient.Get(ctx, types.NamespacedName{Name: oauthServiceNameFromCluster(&foundRayCluster), Namespace: foundRayCluster.Namespace}, &corev1.Service{}) + It("should create all oauth resources", func(ctx SpecContext) { + foundRayCluster, err := rayClient.RayV1().RayClusters(namespaceName).Get(ctx, rayClusterName, metav1.GetOptions{}) Expect(err).To(Not(HaveOccurred())) - err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &corev1.ServiceAccount{}) - Expect(err).To(Not(HaveOccurred())) - err = k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{}) - Expect(err).To(Not(HaveOccurred())) - err = k8sClient.Get(ctx, types.NamespacedName{Name: foundRayCluster.Name, Namespace: foundRayCluster.Namespace}, &routev1.Route{}) + + Eventually(func() (*corev1.Secret, error) { + return k8sClient.CoreV1().Secrets(namespaceName).Get(ctx, oauthSecretNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).ShouldNot(BeNil()) + Eventually(func() (*corev1.Service, error) { + return k8sClient.CoreV1().Services(namespaceName).Get(ctx, oauthServiceNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).ShouldNot(BeNil()) + Eventually(func() (*corev1.ServiceAccount, error) { + return k8sClient.CoreV1().ServiceAccounts(namespaceName).Get(ctx, oauthServiceAccountNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).ShouldNot(BeNil()) + Eventually(func() (*rbacv1.ClusterRoleBinding, error) { + return k8sClient.RbacV1().ClusterRoleBindings().Get(ctx, crbNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).ShouldNot(BeNil()) + Eventually(func() (*routev1.Route, error) { + return routeClient.RouteV1().Routes(namespaceName).Get(ctx, dashboardNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).ShouldNot(BeNil()) + }) + + It("should set owner references for all resources", func(ctx SpecContext) { + foundRayCluster, err := rayClient.RayV1().RayClusters(namespaceName).Get(ctx, rayClusterName, metav1.GetOptions{}) Expect(err).To(Not(HaveOccurred())) + + Eventually(func() (*corev1.Secret, error) { + return k8sClient.CoreV1().Secrets(namespaceName).Get(ctx, oauthSecretNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceKind, Equal("RayCluster"))) + Eventually(func() (*corev1.Secret, error) { + return k8sClient.CoreV1().Secrets(namespaceName).Get(ctx, oauthSecretNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceName, Equal(foundRayCluster.Name))) + Eventually(func() (*corev1.Service, error) { + return k8sClient.CoreV1().Services(namespaceName).Get(ctx, oauthServiceNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceKind, Equal("RayCluster"))) + Eventually(func() (*corev1.Service, error) { + return k8sClient.CoreV1().Services(namespaceName).Get(ctx, oauthServiceNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceName, Equal(foundRayCluster.Name))) + Eventually(func() (*corev1.ServiceAccount, error) { + return k8sClient.CoreV1().ServiceAccounts(namespaceName).Get(ctx, oauthServiceAccountNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceKind, Equal("RayCluster"))) + Eventually(func() (*corev1.ServiceAccount, error) { + return k8sClient.CoreV1().ServiceAccounts(namespaceName).Get(ctx, oauthServiceAccountNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceName, Equal(foundRayCluster.Name))) + Eventually(func() (*routev1.Route, error) { + return routeClient.RouteV1().Routes(namespaceName).Get(ctx, dashboardNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceKind, Equal("RayCluster"))) + Eventually(func() (*routev1.Route, error) { + return routeClient.RouteV1().Routes(namespaceName).Get(ctx, dashboardNameFromCluster(foundRayCluster), metav1.GetOptions{}) + }).WithTimeout(time.Second * 10).Should(WithTransform(OwnerReferenceName, Equal(foundRayCluster.Name))) }) - It("should remove CRB when the RayCluster is deleted", func() { - foundRayCluster := rayv1.RayCluster{} - err := k8sClient.Get(ctx, typeNamespaceName, &foundRayCluster) + It("should remove CRB when the RayCluster is deleted", func(ctx SpecContext) { + foundRayCluster, err := rayClient.RayV1().RayClusters(namespaceName).Get(ctx, rayClusterName, metav1.GetOptions{}) Expect(err).To(Not(HaveOccurred())) - err = k8sClient.Delete(ctx, &foundRayCluster) + + err = rayClient.RayV1().RayClusters(namespaceName).Delete(ctx, foundRayCluster.Name, metav1.DeleteOptions{}) Expect(err).To(Not(HaveOccurred())) - Eventually(func() bool { - return errors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: crbNameFromCluster(&foundRayCluster)}, &rbacv1.ClusterRoleBinding{})) - }, SpecTimeout(time.Second*10)).Should(Equal(true)) + + Eventually(func() error { + _, err := k8sClient.RbacV1().ClusterRoleBindings().Get(ctx, crbNameFromCluster(foundRayCluster), metav1.GetOptions{}) + return err + }).WithTimeout(time.Second * 10).Should(Satisfy(errors.IsNotFound)) }) + }) }) + +func OwnerReferenceKind(meta metav1.Object) string { + return meta.GetOwnerReferences()[0].Kind +} + +func OwnerReferenceName(meta metav1.Object) string { + return meta.GetOwnerReferences()[0].Name +} diff --git a/pkg/controllers/suite_test.go b/pkg/controllers/suite_test.go index 0110179f..6828bdad 100644 --- a/pkg/controllers/suite_test.go +++ b/pkg/controllers/suite_test.go @@ -27,17 +27,18 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" routev1 "github.com/openshift/api/route/v1" + routeclient "github.com/openshift/client-go/route/clientset/versioned" //+kubebuilder:scaffold:imports ) @@ -45,8 +46,12 @@ import ( // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. var cfg *rest.Config -var k8sClient client.Client +var k8sClient *kubernetes.Clientset +var rayClient *rayclient.Clientset +var routeClient *routeclient.Clientset var testEnv *envtest.Environment +var ctx context.Context +var cancel context.CancelFunc func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) @@ -56,10 +61,11 @@ func TestAPIs(t *testing.T) { const ( RayClusterCRDFileDownload = "https://raw.githubusercontent.com/ray-project/kuberay/master/ray-operator/config/crd/bases/ray.io_rayclusters.yaml" - RouteCRDFileDownload = "https://raw.githubusercontent.com/openshift/api/master/route/v1/route.crd.yaml" + RouteCRDFileDownload = "https://raw.githubusercontent.com/openshift/api/master/route/v1/zz_generated.crd-manifests/routes-Default.crd.yaml" ) var _ = BeforeSuite(func() { + ctx, cancel = context.WithCancel(context.Background()) logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) var err error @@ -86,7 +92,6 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{ - filepath.Join("..", "config", "crd"), filepath.Join(".", "test-crds"), }, ErrorIfCRDPathMissing: true, @@ -99,29 +104,31 @@ var _ = BeforeSuite(func() { //+kubebuilder:scaffold:scheme - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + k8sClient, err = kubernetes.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) - clientSet, err := kubernetes.NewForConfig(cfg) + rayClient, err = rayclient.NewForConfig(cfg) + Expect(err).To(Not(HaveOccurred())) + routeClient, err = routeclient.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) err = rayv1.AddToScheme(scheme.Scheme) Expect(err).To(Not(HaveOccurred())) err = routev1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme, }) Expect(err).NotTo(HaveOccurred()) err = (&RayClusterReconciler{ - Client: k8sManager.GetClient(), - Scheme: k8sManager.GetScheme(), - kubeClient: clientSet, - CookieSalt: "foo", + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + CookieSalt: "foo", + IsOpenShift: true, }).SetupWithManager(k8sManager) Expect(err).NotTo(HaveOccurred()) go func() { defer GinkgoRecover() - err = k8sManager.Start(context.Background()) + err = k8sManager.Start(ctx) Expect(err).ToNot(HaveOccurred(), "failed to run manager") }() }) @@ -130,6 +137,7 @@ var _ = AfterSuite(func() { By("tearing down the test environment") err := os.RemoveAll("./test-crds") Expect(err).NotTo(HaveOccurred()) + cancel() err = testEnv.Stop() Expect(err).NotTo(HaveOccurred()) }) From 0873fc4058245c59495915804fbcbbea791439c0 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Wed, 17 Apr 2024 18:01:35 +0100 Subject: [PATCH 343/377] Create RayCluster ValidatingWebhook --- PROJECT | 1 + config/openshift/kustomization.yaml | 4 +- ...ch.yaml => webhookcainjection_mpatch.yaml} | 0 .../openshift/webhookcainjection_vpatch.yaml | 7 ++++ config/webhook/kustomization.yaml | 2 + config/webhook/manifests.yaml | 27 +++++++++++++ pkg/controllers/raycluster_webhook.go | 39 +++++++++++++++++++ 7 files changed, 78 insertions(+), 2 deletions(-) rename config/openshift/{webhookcainjection_patch.yaml => webhookcainjection_mpatch.yaml} (100%) create mode 100644 config/openshift/webhookcainjection_vpatch.yaml diff --git a/PROJECT b/PROJECT index 0fc5562e..965a62fd 100644 --- a/PROJECT +++ b/PROJECT @@ -19,5 +19,6 @@ resources: version: v1 webhooks: defaulting: true + validation: true webhookVersion: v1 version: "3" diff --git a/config/openshift/kustomization.yaml b/config/openshift/kustomization.yaml index 22e16816..b39da073 100644 --- a/config/openshift/kustomization.yaml +++ b/config/openshift/kustomization.yaml @@ -6,7 +6,6 @@ namespace: openshift-operators # "wordpress" becomes "alices-wordpress". # Note that it should also match with the prefix (text before '-') of the namespace # field above. -namePrefix: codeflare-operator- # Labels to add to all resources and selectors. commonLabels: @@ -19,4 +18,5 @@ bases: patches: - path: manager_webhook_patch.yaml -- path: webhookcainjection_patch.yaml +- path: webhookcainjection_mpatch.yaml +- path: webhookcainjection_vpatch.yaml diff --git a/config/openshift/webhookcainjection_patch.yaml b/config/openshift/webhookcainjection_mpatch.yaml similarity index 100% rename from config/openshift/webhookcainjection_patch.yaml rename to config/openshift/webhookcainjection_mpatch.yaml diff --git a/config/openshift/webhookcainjection_vpatch.yaml b/config/openshift/webhookcainjection_vpatch.yaml new file mode 100644 index 00000000..91f741a0 --- /dev/null +++ b/config/openshift/webhookcainjection_vpatch.yaml @@ -0,0 +1,7 @@ +# This patch add annotation to admission webhook config +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + service.beta.openshift.io/inject-cabundle: "true" diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml index 9cf26134..412cee11 100644 --- a/config/webhook/kustomization.yaml +++ b/config/webhook/kustomization.yaml @@ -1,3 +1,5 @@ +namePrefix: codeflare-operator- + resources: - manifests.yaml - service.yaml diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index faf91696..c14976cf 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -25,3 +25,30 @@ webhooks: resources: - rayclusters sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + creationTimestamp: null + name: validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-ray-io-v1-raycluster + failurePolicy: Fail + name: vraycluster.kb.io + rules: + - apiGroups: + - ray.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - rayclusters + sideEffects: None diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 0cd63504..d0c04694 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -23,10 +23,12 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/project-codeflare/codeflare-operator/pkg/config" ) @@ -40,16 +42,24 @@ func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConf WithDefaulter(&rayClusterDefaulter{ Config: cfg, }). + WithValidator(&rayClusterWebhook{ + Config: cfg, + }). Complete() } // +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/validate-ray-io-v1-raycluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=vraycluster.kb.io,admissionReviewVersions=v1 type rayClusterDefaulter struct { Config *config.KubeRayConfiguration } +type rayClusterWebhook struct { + Config *config.KubeRayConfiguration +} var _ webhook.CustomDefaulter = &rayClusterDefaulter{} +var _ webhook.CustomValidator = &rayClusterWebhook{} // Default implements webhook.Defaulter so a webhook will be registered for the type func (r *rayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { @@ -132,3 +142,32 @@ func (r *rayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e return nil } + +func (v *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + raycluster := obj.(*rayv1.RayCluster) + var warnings admission.Warnings + var allErrors field.ErrorList + specPath := field.NewPath("spec") + + if pointer.BoolDeref(raycluster.Spec.HeadGroupSpec.EnableIngress, false) { + rayclusterlog.Info("Creating RayCluster resources with EnableIngress set to true or unspecified is not allowed") + allErrors = append(allErrors, field.Invalid(specPath.Child("headGroupSpec").Child("enableIngress"), raycluster.Spec.HeadGroupSpec.EnableIngress, "creating RayCluster resources with EnableIngress set to true or unspecified is not allowed")) + } + + return warnings, allErrors.ToAggregate() +} + +func (v *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + newRayCluster := newObj.(*rayv1.RayCluster) + if !newRayCluster.DeletionTimestamp.IsZero() { + // Object is being deleted, skip validations + return nil, nil + } + warnings, err := v.ValidateCreate(ctx, newRayCluster) + return warnings, err +} + +func (v *rayClusterWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + // Optional: Add delete validation logic here + return nil, nil +} From b4464d1c1355d475e465469b7bc8afa5c6f64f66 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 18 Apr 2024 10:16:15 +0100 Subject: [PATCH 344/377] Remove Defaulter type in webhook --- pkg/controllers/raycluster_webhook.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index d0c04694..bc003d4d 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -27,7 +27,6 @@ import ( "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/project-codeflare/codeflare-operator/pkg/config" @@ -39,7 +38,7 @@ var rayclusterlog = logf.Log.WithName("raycluster-resource") func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConfiguration) error { return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). - WithDefaulter(&rayClusterDefaulter{ + WithDefaulter(&rayClusterWebhook{ Config: cfg, }). WithValidator(&rayClusterWebhook{ @@ -51,21 +50,15 @@ func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConf // +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 // +kubebuilder:webhook:path=/validate-ray-io-v1-raycluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=vraycluster.kb.io,admissionReviewVersions=v1 -type rayClusterDefaulter struct { - Config *config.KubeRayConfiguration -} type rayClusterWebhook struct { Config *config.KubeRayConfiguration } -var _ webhook.CustomDefaulter = &rayClusterDefaulter{} -var _ webhook.CustomValidator = &rayClusterWebhook{} - // Default implements webhook.Defaulter so a webhook will be registered for the type -func (r *rayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error { +func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error { raycluster := obj.(*rayv1.RayCluster) - if !pointer.BoolDeref(r.Config.RayDashboardOAuthEnabled, true) { + if !pointer.BoolDeref(w.Config.RayDashboardOAuthEnabled, true) { return nil } @@ -143,7 +136,7 @@ func (r *rayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e return nil } -func (v *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { +func (w *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { raycluster := obj.(*rayv1.RayCluster) var warnings admission.Warnings var allErrors field.ErrorList @@ -157,17 +150,17 @@ func (v *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Obje return warnings, allErrors.ToAggregate() } -func (v *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { +func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { newRayCluster := newObj.(*rayv1.RayCluster) if !newRayCluster.DeletionTimestamp.IsZero() { // Object is being deleted, skip validations return nil, nil } - warnings, err := v.ValidateCreate(ctx, newRayCluster) + warnings, err := w.ValidateCreate(ctx, newRayCluster) return warnings, err } -func (v *rayClusterWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { +func (w *rayClusterWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { // Optional: Add delete validation logic here return nil, nil } From 2b885ebf330b29d64deb50829c13eaa95b1cef1f Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 18 Apr 2024 10:32:44 +0100 Subject: [PATCH 345/377] Use single rayClusterWebhook instance --- pkg/controllers/raycluster_webhook.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index bc003d4d..8d06eebe 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -27,6 +27,7 @@ import ( "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/project-codeflare/codeflare-operator/pkg/config" @@ -36,14 +37,13 @@ import ( var rayclusterlog = logf.Log.WithName("raycluster-resource") func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConfiguration) error { + rayClusterWebhookInstance := &rayClusterWebhook{ + Config: cfg, + } return ctrl.NewWebhookManagedBy(mgr). For(&rayv1.RayCluster{}). - WithDefaulter(&rayClusterWebhook{ - Config: cfg, - }). - WithValidator(&rayClusterWebhook{ - Config: cfg, - }). + WithDefaulter(rayClusterWebhookInstance). + WithValidator(rayClusterWebhookInstance). Complete() } @@ -54,6 +54,9 @@ type rayClusterWebhook struct { Config *config.KubeRayConfiguration } +var _ webhook.CustomDefaulter = &rayClusterWebhook{} +var _ webhook.CustomValidator = &rayClusterWebhook{} + // Default implements webhook.Defaulter so a webhook will be registered for the type func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error { raycluster := obj.(*rayv1.RayCluster) From a51f336b2f4a8271d519bb46ccf89a459061dd58 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 18 Apr 2024 12:09:24 +0200 Subject: [PATCH 346/377] Fix oauth-proxy cookie Secret --- pkg/controllers/raycluster_webhook.go | 49 ++++++++++++--------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 8d06eebe..d465fd2e 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -59,14 +59,14 @@ var _ webhook.CustomValidator = &rayClusterWebhook{} // Default implements webhook.Defaulter so a webhook will be registered for the type func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error { - raycluster := obj.(*rayv1.RayCluster) + rayCluster := obj.(*rayv1.RayCluster) if !pointer.BoolDeref(w.Config.RayDashboardOAuthEnabled, true) { return nil } // Check and add OAuth proxy if it does not exist - for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers { + for _, container := range rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers { if container.Name == "oauth-proxy" { rayclusterlog.V(2).Info("OAuth sidecar already exists, no patch needed") return nil @@ -74,17 +74,30 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err } rayclusterlog.V(2).Info("Adding OAuth sidecar container") - // definition of the new container + newOAuthSidecar := corev1.Container{ Name: "oauth-proxy", Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", Ports: []corev1.ContainerPort{ {ContainerPort: 8443, Name: "oauth-proxy"}, }, + Env: []corev1.EnvVar{ + { + Name: "COOKIE_SECRET", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: rayCluster.Name + "-oauth-config", + }, + Key: "cookie_secret", + }, + }, + }, + }, Args: []string{ "--https-address=:8443", "--provider=openshift", - "--openshift-service-account=" + raycluster.Name + "-oauth-proxy", + "--openshift-service-account=" + rayCluster.Name + "-oauth-proxy", "--upstream=http://localhost:8265", "--tls-cert=/etc/tls/private/tls.crt", "--tls-key=/etc/tls/private/tls.key", @@ -100,40 +113,22 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err }, } - // Adding the new OAuth sidecar container - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) - - cookieSecret := corev1.EnvVar{ - Name: "COOKIE_SECRET", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: raycluster.Name + "-oauth-config", - }, - Key: "cookie_secret", - }, - }, - } - - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = append( - raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, - cookieSecret, - ) + rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) tlsSecretVolume := corev1.Volume{ Name: "proxy-tls-secret", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: raycluster.Name + "-proxy-tls-secret", + SecretName: rayCluster.Name + "-proxy-tls-secret", }, }, } - raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) + rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) // Ensure the service account is set - if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { - raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy" + if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { + rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy" } return nil From 1c85429226d05c74d3857c384f1ee37c8f9bd673 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 18 Apr 2024 14:57:03 +0200 Subject: [PATCH 347/377] Use deep semantic comparison in RayCluster validation webhook --- config/webhook/manifests.yaml | 1 - pkg/controllers/raycluster_webhook.go | 160 +++++++++++++++++--------- pkg/controllers/support.go | 56 ++++++++- 3 files changed, 158 insertions(+), 59 deletions(-) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index c14976cf..ee680a18 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -21,7 +21,6 @@ webhooks: - v1 operations: - CREATE - - UPDATE resources: - rayclusters sideEffects: None diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index d465fd2e..4ac908b5 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -33,6 +33,11 @@ import ( "github.com/project-codeflare/codeflare-operator/pkg/config" ) +const ( + oauthProxyContainerName = "oauth-proxy" + oauthProxyVolumeName = "proxy-tls-secret" +) + // log is for logging in this package. var rayclusterlog = logf.Log.WithName("raycluster-resource") @@ -47,7 +52,7 @@ func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConf Complete() } -// +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 // +kubebuilder:webhook:path=/validate-ray-io-v1-raycluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=vraycluster.kb.io,admissionReviewVersions=v1 type rayClusterWebhook struct { @@ -65,18 +70,105 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err return nil } - // Check and add OAuth proxy if it does not exist - for _, container := range rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers { - if container.Name == "oauth-proxy" { - rayclusterlog.V(2).Info("OAuth sidecar already exists, no patch needed") - return nil - } + rayclusterlog.V(2).Info("Adding OAuth sidecar container") + + rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, oauthProxyContainer(rayCluster), withContainerName(oauthProxyContainerName)) + + rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), withVolumeName(oauthProxyVolumeName)) + + rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy" + + return nil +} + +func (w *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + rayCluster := obj.(*rayv1.RayCluster) + + var warnings admission.Warnings + var allErrors field.ErrorList + + allErrors = append(allErrors, validateIngress(rayCluster)...) + + return warnings, allErrors.ToAggregate() +} + +func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + rayCluster := newObj.(*rayv1.RayCluster) + + var warnings admission.Warnings + var allErrors field.ErrorList + + if !rayCluster.DeletionTimestamp.IsZero() { + // Object is being deleted, skip validations + return nil, nil + } + + allErrors = append(allErrors, validateIngress(rayCluster)...) + allErrors = append(allErrors, validateOAuthProxyContainer(rayCluster)...) + allErrors = append(allErrors, validateOAuthProxyVolume(rayCluster)...) + allErrors = append(allErrors, validateHeadGroupServiceAccountName(rayCluster)...) + + return warnings, allErrors.ToAggregate() +} + +func (w *rayClusterWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + // Optional: Add delete validation logic here + return nil, nil +} + +func validateOAuthProxyContainer(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, oauthProxyContainer(rayCluster), byContainerName, + field.NewPath("spec", "headGroupSpec", "template", "spec", "containers"), + "OAuth Proxy container is immutable"); err != nil { + allErrors = append(allErrors, err) } - rayclusterlog.V(2).Info("Adding OAuth sidecar container") + return allErrors +} + +func validateOAuthProxyVolume(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList - newOAuthSidecar := corev1.Container{ - Name: "oauth-proxy", + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), byVolumeName, + field.NewPath("spec", "headGroupSpec", "template", "spec", "volumes"), + "OAuth Proxy TLS Secret volume is immutable"); err != nil { + allErrors = append(allErrors, err) + } + + return allErrors +} + +func validateIngress(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + if pointer.BoolDeref(rayCluster.Spec.HeadGroupSpec.EnableIngress, false) { + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "headGroupSpec", "enableIngress"), + rayCluster.Spec.HeadGroupSpec.EnableIngress, + "RayCluster resources with EnableIngress set to true or unspecified is not allowed")) + } + + return allErrors +} + +func validateHeadGroupServiceAccountName(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName != rayCluster.Name+"-oauth-proxy" { + allErrors = append(allErrors, field.Invalid( + field.NewPath("spec", "headGroupSpec", "template", "spec", "serviceAccountName"), + rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName, + "RayCluster head group service account is immutable")) + } + + return allErrors +} + +func oauthProxyContainer(rayCluster *rayv1.RayCluster) corev1.Container { + return corev1.Container{ + Name: oauthProxyContainerName, Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366", Ports: []corev1.ContainerPort{ {ContainerPort: 8443, Name: "oauth-proxy"}, @@ -106,59 +198,21 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err }, VolumeMounts: []corev1.VolumeMount{ { - Name: "proxy-tls-secret", + Name: oauthProxyVolumeName, MountPath: "/etc/tls/private", ReadOnly: true, }, }, } +} - rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar) - - tlsSecretVolume := corev1.Volume{ - Name: "proxy-tls-secret", +func oauthProxyTLSSecretVolume(rayCluster *rayv1.RayCluster) corev1.Volume { + return corev1.Volume{ + Name: oauthProxyVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: rayCluster.Name + "-proxy-tls-secret", }, }, } - - rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume) - - // Ensure the service account is set - if rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" { - rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy" - } - - return nil -} - -func (w *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { - raycluster := obj.(*rayv1.RayCluster) - var warnings admission.Warnings - var allErrors field.ErrorList - specPath := field.NewPath("spec") - - if pointer.BoolDeref(raycluster.Spec.HeadGroupSpec.EnableIngress, false) { - rayclusterlog.Info("Creating RayCluster resources with EnableIngress set to true or unspecified is not allowed") - allErrors = append(allErrors, field.Invalid(specPath.Child("headGroupSpec").Child("enableIngress"), raycluster.Spec.HeadGroupSpec.EnableIngress, "creating RayCluster resources with EnableIngress set to true or unspecified is not allowed")) - } - - return warnings, allErrors.ToAggregate() -} - -func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { - newRayCluster := newObj.(*rayv1.RayCluster) - if !newRayCluster.DeletionTimestamp.IsZero() { - // Object is being deleted, skip validations - return nil, nil - } - warnings, err := w.ValidateCreate(ctx, newRayCluster) - return warnings, err -} - -func (w *rayClusterWebhook) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { - // Optional: Add delete validation logic here - return nil, nil } diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 673d5941..0efbc747 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -3,9 +3,11 @@ package controllers import ( rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" - "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/validation/field" v1 "k8s.io/client-go/applyconfigurations/meta/v1" networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" @@ -29,7 +31,6 @@ func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConf ) } -// Create an Ingress object for the RayCluster func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration { return networkingv1ac.Ingress(rayClientNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). @@ -42,7 +43,7 @@ func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *net WithAPIVersion(cluster.APIVersion). WithKind(cluster.Kind). WithName(cluster.Name). - WithUID(types.UID(cluster.UID))). + WithUID(cluster.UID)). WithSpec(networkingv1ac.IngressSpec(). WithIngressClassName("nginx"). WithRules(networkingv1ac.IngressRule(). @@ -65,7 +66,6 @@ func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressHost string) *net ) } -// Create an Ingress object for the RayCluster func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration { return networkingv1ac.Ingress(dashboardNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). @@ -73,7 +73,7 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo WithAPIVersion(cluster.APIVersion). WithKind(cluster.Kind). WithName(cluster.Name). - WithUID(types.UID(cluster.UID))). + WithUID(cluster.UID)). WithSpec(networkingv1ac.IngressSpec(). WithRules(networkingv1ac.IngressRule(). WithHost(ingressHost). // Full Hostname @@ -94,3 +94,49 @@ func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *netwo ), ) } + +type compare[T any] func(T, T) bool + +func upsert[T any](items []T, item T, predicate compare[T]) []T { + for i, t := range items { + if predicate(t, item) { + items[i] = item + return items + } + } + return append(items, item) +} + +func contains[T any](items []T, item T, predicate compare[T], path *field.Path, msg string) *field.Error { + for _, t := range items { + if predicate(t, item) { + if equality.Semantic.DeepDerivative(item, t) { + return nil + } + return field.Invalid(path, t, msg) + } + } + return field.Required(path, msg) +} + +var byContainerName = compare[corev1.Container]( + func(c1, c2 corev1.Container) bool { + return c1.Name == c2.Name + }) + +func withContainerName(name string) compare[corev1.Container] { + return func(c1, c2 corev1.Container) bool { + return c1.Name == name + } +} + +var byVolumeName = compare[corev1.Volume]( + func(v1, v2 corev1.Volume) bool { + return v1.Name == v2.Name + }) + +func withVolumeName(name string) compare[corev1.Volume] { + return func(v1, v2 corev1.Volume) bool { + return v1.Name == name + } +} From 1518c686e7f7b55f3803c3ca457ef335bb197c68 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 18 Apr 2024 15:19:42 +0200 Subject: [PATCH 348/377] Rename RayCluster webhooks --- config/webhook/manifests.yaml | 4 ++-- pkg/controllers/raycluster_webhook.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index ee680a18..5e98107c 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -13,7 +13,7 @@ webhooks: namespace: system path: /mutate-ray-io-v1-raycluster failurePolicy: Fail - name: mraycluster.kb.io + name: mraycluster.ray.openshift.ai rules: - apiGroups: - ray.io @@ -39,7 +39,7 @@ webhooks: namespace: system path: /validate-ray-io-v1-raycluster failurePolicy: Fail - name: vraycluster.kb.io + name: vraycluster.ray.openshift.ai rules: - apiGroups: - ray.io diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 4ac908b5..e7d9a081 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -52,8 +52,8 @@ func SetupRayClusterWebhookWithManager(mgr ctrl.Manager, cfg *config.KubeRayConf Complete() } -// +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1 -// +kubebuilder:webhook:path=/validate-ray-io-v1-raycluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=vraycluster.kb.io,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create,versions=v1,name=mraycluster.ray.openshift.ai,admissionReviewVersions=v1 +// +kubebuilder:webhook:path=/validate-ray-io-v1-raycluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=vraycluster.ray.openshift.ai,admissionReviewVersions=v1 type rayClusterWebhook struct { Config *config.KubeRayConfiguration From 7c644080f71e7cd5a7a87ff7863220fd3b645aad Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 18 Apr 2024 15:24:32 +0200 Subject: [PATCH 349/377] Better message for RayCluster enableIngress field validation --- pkg/controllers/raycluster_webhook.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index e7d9a081..27a4b56b 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -147,7 +147,7 @@ func validateIngress(rayCluster *rayv1.RayCluster) field.ErrorList { allErrors = append(allErrors, field.Invalid( field.NewPath("spec", "headGroupSpec", "enableIngress"), rayCluster.Spec.HeadGroupSpec.EnableIngress, - "RayCluster resources with EnableIngress set to true or unspecified is not allowed")) + "RayCluster with enableIngress set to true is not allowed")) } return allErrors From b90451c16b693a1159ad846ad71129fe07002f0e Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 18 Apr 2024 18:11:07 -0400 Subject: [PATCH 350/377] Get go version for setup-go action from go.mod file --- .github/workflows/component_tests.yaml | 2 +- .github/workflows/e2e_tests.yaml | 2 +- .github/workflows/mnist-job-test-image.yml | 2 +- .github/workflows/olm_tests.yaml | 2 +- .github/workflows/operator-image.yml | 2 +- .github/workflows/precommit.yml | 2 +- .github/workflows/tag-and-build.yml | 2 +- .github/workflows/unit_tests.yml | 2 +- .github/workflows/verify_generated_files.yml | 4 ++-- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/component_tests.yaml b/.github/workflows/component_tests.yaml index 815025b2..8f5bf7a6 100644 --- a/.github/workflows/component_tests.yaml +++ b/.github/workflows/component_tests.yaml @@ -36,7 +36,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Run component tests run: | diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml index 30ef1ea9..0a0ad397 100644 --- a/.github/workflows/e2e_tests.yaml +++ b/.github/workflows/e2e_tests.yaml @@ -45,7 +45,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 diff --git a/.github/workflows/mnist-job-test-image.yml b/.github/workflows/mnist-job-test-image.yml index 4323bd5d..02c4b5c7 100644 --- a/.github/workflows/mnist-job-test-image.yml +++ b/.github/workflows/mnist-job-test-image.yml @@ -19,7 +19,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Login to Quay.io id: podman-login-quay diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml index 472eda20..abce29d3 100644 --- a/.github/workflows/olm_tests.yaml +++ b/.github/workflows/olm_tests.yaml @@ -44,7 +44,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Set up gotestfmt uses: gotesttools/gotestfmt-action@v2 diff --git a/.github/workflows/operator-image.yml b/.github/workflows/operator-image.yml index 8c30a8c1..0713d36c 100644 --- a/.github/workflows/operator-image.yml +++ b/.github/workflows/operator-image.yml @@ -22,7 +22,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Login to Quay.io id: podman-login-quay diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 2ca5869c..267d537a 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -26,7 +26,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Activate cache uses: actions/cache@v4 diff --git a/.github/workflows/tag-and-build.yml b/.github/workflows/tag-and-build.yml index a60dceb7..dc4a07a2 100644 --- a/.github/workflows/tag-and-build.yml +++ b/.github/workflows/tag-and-build.yml @@ -56,7 +56,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Verify that release doesn't exist yet shell: bash {0} diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 747cee03..b8e39f08 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -27,7 +27,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Activate cache uses: actions/cache@v4 diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml index 9027c1aa..a5f6eb38 100644 --- a/.github/workflows/verify_generated_files.yml +++ b/.github/workflows/verify_generated_files.yml @@ -24,7 +24,7 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Verify that imports are organized run: make verify-imports @@ -35,6 +35,6 @@ jobs: - name: Set Go uses: actions/setup-go@v5 with: - go-version: v1.20 + go-version-file: './go.mod' - name: Verify that the latest WebhookConfigurations, ClusterRoles, and CustomResourceDefinitions have been generated run: make manifests && git diff --exit-code From abdebf594e980d76d9846644636aa56adb7f156f Mon Sep 17 00:00:00 2001 From: David Grove Date: Thu, 18 Apr 2024 18:21:06 -0400 Subject: [PATCH 351/377] Remove dangling MCAD and InstaScale references Cleanup a few MCAD/InstaScale references missed in c771aae4. --- CONTRIBUTING.md | 10 ---------- README.md | 8 ++------ .../codeflare-operator.clusterserviceversion.yaml | 2 -- main.go | 12 ++++-------- 4 files changed, 6 insertions(+), 26 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8613b92f..3d7bf8d3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,15 +14,6 @@ The following should be installed in your working environment: ## Basic Overview The main entrypoint for the operator is `main.go` -The MCAD and InstaScale custom resources are defined under the `api` dir: - - See `mcad_types.go` and `instascale_types.go` - -The MCAD and InstaScale resource templates can be found under `config/internal`: - - Sorted under `mcad` and `instascale` subdirs - -The code for MCAD/InstaScale resource reconcilliation can be found in the `controllers` dir: - - See `mcad_controller.go` and `instascale_controller.go` - ## Building and Deployment If changes are made in the `api` dir, run: `make manifests` - This will generate new CRDs and associated files @@ -53,7 +44,6 @@ The CodeFlare Operator currently has unit tests and pre-commit checks - Note that both are required for CI to pass on pull requests To write and inspect unit tests: - - MCAD and InstaScale unit tests under `mcad_controller_test.go` and `instascale_controller_test.go` in the `controllers` dir - Unit test functions are defined in `suite_test.go` (with utils in `util/util.go`) in the `controllers dir` - Test cases defined under `controllers/testdata` diff --git a/README.md b/README.md index ef2b44d5..423fb974 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # codeflare-operator -Operator for installation and lifecycle management of CodeFlare distributed workload stack, starting with MCAD and InstaScale +Operator for installation and lifecycle management of CodeFlare distributed workload stack. @@ -9,9 +9,7 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| | CodeFlare Operator | [v1.3.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.3.1) | -| Multi-Cluster App Dispatcher | [v1.40.0](https://github.com/project-codeflare/multi-cluster-app-dispatcher/releases/tag/v1.40.0) | | CodeFlare-SDK | [v0.15.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.15.1) | -| InstaScale | [v0.4.0](https://github.com/project-codeflare/instascale/releases/tag/v0.4.0) | | KubeRay | [v1.0.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0) | @@ -90,7 +88,7 @@ The e2e tests can be executed locally by running the following commands: To properly run e2e tests on disconnected cluster user has to provide additional environment variables to properly configure testing environment: -- `CODEFLARE_TEST_PYTORCH_IMAGE` - image tag for image used to run training job using MCAD +- `CODEFLARE_TEST_PYTORCH_IMAGE` - image tag for image used to run training job - `CODEFLARE_TEST_RAY_IMAGE` - image tag for Ray cluster image - `MNIST_DATASET_URL` - URL where MNIST dataset is available - `PIP_INDEX_URL` - URL where PyPI server with needed dependencies is running @@ -118,8 +116,6 @@ For ODH tests additional environment variables are needed: There may be instances in which a new CodeFlare stack release requires releases of only a subset of the stack components. Examples could be hotfixes for a specific component. In these instances: 1. Build updated components as needed: - - Build and release [MCAD](https://github.com/project-codeflare/multi-cluster-app-dispatcher) - - Build and release [InstaScale](https://github.com/project-codeflare/instascale) - Build and release [CodeFlare-SDK](https://github.com/project-codeflare/codeflare-sdk) 2. Invoke [tag-and-build.yml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/tag-and-build.yml) GitHub action, this action will create a repository tag, build and push operator image. diff --git a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml index 04a0b301..f47cdd1a 100644 --- a/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/codeflare-operator.clusterserviceversion.yaml @@ -72,9 +72,7 @@ spec: keywords: - Pipelines - Scaling - - MCAD - App - - InstaScale - Jobs links: - name: Codeflare Operator diff --git a/main.go b/main.go index 7eff5fc4..b1aa0a91 100644 --- a/main.go +++ b/main.go @@ -56,12 +56,10 @@ import ( ) var ( - scheme = runtime.NewScheme() - setupLog = ctrl.Log.WithName("setup") - OperatorVersion = "UNKNOWN" - McadVersion = "UNKNOWN" - InstaScaleVersion = "UNKNOWN" - BuildDate = "UNKNOWN" + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + OperatorVersion = "UNKNOWN" + BuildDate = "UNKNOWN" ) func init() { @@ -90,8 +88,6 @@ func main() { setupLog.Info("Build info", "operatorVersion", OperatorVersion, - "mcadVersion", McadVersion, - "instaScaleVersion", InstaScaleVersion, "date", BuildDate, ) From a0edab0385ea96f484878f01292b4c059033f587 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Thu, 18 Apr 2024 19:52:21 +0200 Subject: [PATCH 352/377] Use cert-controller to generate webhook certificates --- config/default/kustomization.yaml | 4 + .../manager_webhook_patch.yaml | 2 +- config/openshift/kustomization.yaml | 22 ---- .../openshift/webhookcainjection_mpatch.yaml | 7 -- .../openshift/webhookcainjection_vpatch.yaml | 7 -- config/rbac/role.yaml | 27 +++++ config/webhook/kustomization.yaml | 3 +- config/webhook/secret.yaml | 5 + config/webhook/service.yaml | 3 - go.mod | 35 +++--- go.sum | 105 +++++++----------- main.go | 86 ++++++++++---- pkg/controllers/raycluster_webhook.go | 6 +- 13 files changed, 167 insertions(+), 145 deletions(-) rename config/{openshift => default}/manager_webhook_patch.yaml (87%) delete mode 100644 config/openshift/kustomization.yaml delete mode 100644 config/openshift/webhookcainjection_mpatch.yaml delete mode 100644 config/openshift/webhookcainjection_vpatch.yaml create mode 100644 config/webhook/secret.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 42fa7179..30b1273c 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -16,8 +16,12 @@ commonLabels: bases: - ../rbac - ../manager + - ../webhook # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. # - ../prometheus resources: - metrics_service.yaml + +patches: + - path: manager_webhook_patch.yaml diff --git a/config/openshift/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml similarity index 87% rename from config/openshift/manager_webhook_patch.yaml rename to config/default/manager_webhook_patch.yaml index 5d6b541d..ab553c00 100644 --- a/config/openshift/manager_webhook_patch.yaml +++ b/config/default/manager_webhook_patch.yaml @@ -20,4 +20,4 @@ spec: - name: cert secret: defaultMode: 420 - secretName: codeflare-operator-raycluster-webhook-cert + secretName: webhook-server-cert diff --git a/config/openshift/kustomization.yaml b/config/openshift/kustomization.yaml deleted file mode 100644 index b39da073..00000000 --- a/config/openshift/kustomization.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Adds namespace to all resources. -namespace: openshift-operators - -# Value of this field is prepended to the -# names of all resources, e.g. a deployment named -# "wordpress" becomes "alices-wordpress". -# Note that it should also match with the prefix (text before '-') of the namespace -# field above. - -# Labels to add to all resources and selectors. -commonLabels: - app.kubernetes.io/name: codeflare-operator - app.kubernetes.io/part-of: codeflare - -bases: - - ../default - - ../webhook - -patches: -- path: manager_webhook_patch.yaml -- path: webhookcainjection_mpatch.yaml -- path: webhookcainjection_vpatch.yaml diff --git a/config/openshift/webhookcainjection_mpatch.yaml b/config/openshift/webhookcainjection_mpatch.yaml deleted file mode 100644 index 39db8634..00000000 --- a/config/openshift/webhookcainjection_mpatch.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# This patch add annotation to admission webhook config -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: mutating-webhook-configuration - annotations: - service.beta.openshift.io/inject-cabundle: "true" diff --git a/config/openshift/webhookcainjection_vpatch.yaml b/config/openshift/webhookcainjection_vpatch.yaml deleted file mode 100644 index 91f741a0..00000000 --- a/config/openshift/webhookcainjection_vpatch.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# This patch add annotation to admission webhook config -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validating-webhook-configuration - annotations: - service.beta.openshift.io/inject-cabundle: "true" diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 70a6a861..24ac7087 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,6 +5,33 @@ metadata: creationTimestamp: null name: manager-role rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - get + - list + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - list + - update + - watch - apiGroups: - authentication.k8s.io resources: diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml index 412cee11..297cf8a8 100644 --- a/config/webhook/kustomization.yaml +++ b/config/webhook/kustomization.yaml @@ -1,8 +1,7 @@ -namePrefix: codeflare-operator- - resources: - manifests.yaml - service.yaml +- secret.yaml configurations: - kustomizeconfig.yaml diff --git a/config/webhook/secret.yaml b/config/webhook/secret.yaml new file mode 100644 index 00000000..af960e6c --- /dev/null +++ b/config/webhook/secret.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Secret +metadata: + name: webhook-server-cert + namespace: system diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml index ada0e328..01a9f23d 100644 --- a/config/webhook/service.yaml +++ b/config/webhook/service.yaml @@ -2,9 +2,6 @@ apiVersion: v1 kind: Service metadata: name: webhook-service - namespace: openshift-operators - annotations: - service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert spec: ports: - port: 443 diff --git a/go.mod b/go.mod index 80f3a52b..06797c21 100644 --- a/go.mod +++ b/go.mod @@ -5,18 +5,19 @@ go 1.20 require ( github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 + github.com/open-policy-agent/cert-controller v0.10.1 github.com/openshift/api v0.0.0-20230213134911-7ba313770556 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a github.com/ray-project/kuberay/ray-operator v1.0.0 go.uber.org/zap v1.26.0 - k8s.io/api v0.27.8 - k8s.io/apimachinery v0.27.8 - k8s.io/client-go v0.27.8 - k8s.io/component-base v0.27.8 - k8s.io/klog/v2 v2.90.1 - k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 - sigs.k8s.io/controller-runtime v0.15.3 + k8s.io/api v0.28.1 + k8s.io/apimachinery v0.28.1 + k8s.io/client-go v0.28.1 + k8s.io/component-base v0.28.1 + k8s.io/klog/v2 v2.100.1 + k8s.io/utils v0.0.0-20230726121419-3b25d923346b + sigs.k8s.io/controller-runtime v0.16.1 sigs.k8s.io/yaml v1.3.0 ) @@ -28,22 +29,22 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.10.1 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.3 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic v0.6.9 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect @@ -66,7 +67,9 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.4 // indirect - go.uber.org/multierr v1.10.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sys v0.16.0 // indirect @@ -74,14 +77,14 @@ require ( golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.12.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.27.7 // indirect - k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect + k8s.io/apiextensions-apiserver v0.28.1 // indirect + k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index 51ec31f4..8a496b7e 100644 --- a/go.sum +++ b/go.sum @@ -34,13 +34,11 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -48,12 +46,10 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -63,8 +59,6 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -73,25 +67,20 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -108,8 +97,8 @@ github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= -github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -121,8 +110,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= -github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= @@ -159,8 +148,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= -github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -188,14 +177,12 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -275,7 +262,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -326,6 +312,9 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5nbVfmstNpj4kHQFkvPK4= +github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= github.com/openshift-online/ocm-sdk-go v0.1.368 h1:qP+gkChV8WDwwpkUw1xUyjTXKdvrwyd70Gff2GMUSeU= github.com/openshift-online/ocm-sdk-go v0.1.368/go.mod h1:KYOw8kAKAHyPrJcQoVR82CneQ4ofC02Na4cXXaTq4Nw= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= @@ -370,9 +359,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -383,10 +371,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -403,9 +389,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -418,20 +401,21 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -463,6 +447,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -523,7 +509,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -618,7 +603,6 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= @@ -686,8 +670,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= -gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -735,14 +719,12 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -755,9 +737,6 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -770,7 +749,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -787,7 +765,6 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -795,7 +772,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -806,27 +782,28 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.27.8 h1:Ja93gbyII5c3TJzWefEwGhlqC5SZksWhzRS+OYHIocU= -k8s.io/api v0.27.8/go.mod h1:2HuWJC6gpx4UScY+ezziNzv6j6Jqd2q0rGgobYSSjcs= -k8s.io/apiextensions-apiserver v0.27.7 h1:YqIOwZAUokzxJIjunmUd4zS1v3JhK34EPXn+pP0/bsU= -k8s.io/apiextensions-apiserver v0.27.7/go.mod h1:x0p+b5a955lfPz9gaDeBy43obM12s+N9dNHK6+dUL+g= -k8s.io/apimachinery v0.27.8 h1:Xg+ogjDm8s7KmV3vZGf7uOZ0jrC6FPy2Lk/h7BIRmvg= -k8s.io/apimachinery v0.27.8/go.mod h1:EIXLxLt/b1muPITiF5zlrcm7I+YnXsIgM+0GdnPTQvA= -k8s.io/client-go v0.27.8 h1:uXIsGniqc85kKQ8FV0iAwspb0JHtS1UybqrqzHaQ9hc= -k8s.io/client-go v0.27.8/go.mod h1:Ka6MUpObn3LRTfFPvjzyettp8LXCbhqLzZfi8TD4fP8= -k8s.io/component-base v0.27.8 h1:O8YRFv/wWvoo9z62p1N52lq+w5FpzILAlE1h8b9o3K8= -k8s.io/component-base v0.27.8/go.mod h1:h3uyZl+bFQeuLRz3owfSLaw3JKTrn6gmbvybkkW2z+I= -k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= -k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= -k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= -k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= +k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg= +k8s.io/apiextensions-apiserver v0.28.1 h1:l2ThkBRjrWpw4f24uq0Da2HaEgqJZ7pcgiEUTKSmQZw= +k8s.io/apiextensions-apiserver v0.28.1/go.mod h1:sVvrI+P4vxh2YBBcm8n2ThjNyzU4BQGilCQ/JAY5kGs= +k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= +k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8= +k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE= +k8s.io/component-base v0.28.1 h1:LA4AujMlK2mr0tZbQDZkjWbdhTV5bRyEyAFe0TJxlWg= +k8s.io/component-base v0.28.1/go.mod h1:jI11OyhbX21Qtbav7JkhehyBsIRfnO8oEgoAR12ArIU= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-aggregator v0.28.1 h1:rvG4llYnQKHjj6YjjoBPEJxfD1uH0DJwkrJTNKGAaCs= +k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 h1:nVKRi5eItf3x9kkIMfdT4D1/LqPzj0bLjxLYWbdUtV0= +k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.15.3 h1:L+t5heIaI3zeejoIyyvLQs5vTVu/67IU2FfisVzFlBc= -sigs.k8s.io/controller-runtime v0.15.3/go.mod h1:kp4jckA4vTx281S/0Yk2LFEEQe67mjg+ev/yknv47Ds= +sigs.k8s.io/controller-runtime v0.16.1 h1:+15lzrmHsE0s2kNl0Dl8cTchI5Cs8qofo5PGcPrV9z0= +sigs.k8s.io/controller-runtime v0.16.1/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= diff --git a/main.go b/main.go index b1aa0a91..4228d1b4 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ import ( "strings" "time" + cert "github.com/open-policy-agent/cert-controller/pkg/rotator" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" "go.uber.org/zap/zapcore" @@ -32,6 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" @@ -40,10 +42,11 @@ import ( "k8s.io/client-go/rest" configv1alpha1 "k8s.io/component-base/config/v1alpha1" "k8s.io/klog/v2" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/yaml" routev1 "github.com/openshift/api/route/v1" @@ -95,8 +98,8 @@ func main() { cfg := &config.CodeFlareOperatorConfiguration{ ClientConnection: &config.ClientConnection{ - QPS: pointer.Float32(50), - Burst: pointer.Int32(100), + QPS: ptr.To(float32(50)), + Burst: ptr.To(int32(100)), }, ControllerManager: config.ControllerManager{ Metrics: config.MetricsConfiguration{ @@ -110,7 +113,7 @@ func main() { LeaderElection: &configv1alpha1.LeaderElectionConfiguration{}, }, KubeRay: &config.KubeRayConfiguration{ - RayDashboardOAuthEnabled: pointer.Bool(true), + RayDashboardOAuthEnabled: ptr.To(true), IngressDomain: "", }, } @@ -123,17 +126,21 @@ func main() { kubeClient, err := kubernetes.NewForConfig(kubeConfig) exitOnError(err, "unable to create Kubernetes client") - exitOnError(loadIntoOrCreate(ctx, kubeClient, namespaceOrDie(), configMapName, cfg), "unable to initialise configuration") + namespace := namespaceOrDie() - kubeConfig.Burst = int(pointer.Int32Deref(cfg.ClientConnection.Burst, int32(rest.DefaultBurst))) - kubeConfig.QPS = pointer.Float32Deref(cfg.ClientConnection.QPS, rest.DefaultQPS) + exitOnError(loadIntoOrCreate(ctx, kubeClient, namespace, configMapName, cfg), "unable to initialise configuration") + + kubeConfig.Burst = int(ptr.Deref(cfg.ClientConnection.Burst, int32(rest.DefaultBurst))) + kubeConfig.QPS = ptr.Deref(cfg.ClientConnection.QPS, rest.DefaultQPS) setupLog.V(2).Info("REST client", "qps", kubeConfig.QPS, "burst", kubeConfig.Burst) mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: cfg.Metrics.BindAddress, + Scheme: scheme, + Metrics: metricsserver.Options{ + BindAddress: cfg.Metrics.BindAddress, + }, HealthProbeBindAddress: cfg.Health.BindAddress, - LeaderElection: pointer.BoolDeref(cfg.LeaderElection.LeaderElect, false), + LeaderElection: ptr.Deref(cfg.LeaderElection.LeaderElect, false), LeaderElectionID: cfg.LeaderElection.ResourceName, LeaderElectionNamespace: cfg.LeaderElection.ResourceNamespace, LeaderElectionResourceLock: cfg.LeaderElection.ResourceLock, @@ -141,16 +148,30 @@ func main() { RetryPeriod: &cfg.LeaderElection.RetryPeriod.Duration, RenewDeadline: &cfg.LeaderElection.RenewDeadline.Duration, }) - exitOnError(err, "unable to start manager") + exitOnError(err, "unable to create manager") + + certsReady := make(chan struct{}) + exitOnError(setupCertManagement(mgr, namespace, certsReady), "unable to setup cert-controller") OpenShift := isOpenShift(ctx, kubeClient.DiscoveryClient) - if OpenShift { - // TODO: setup the RayCluster webhook on vanilla Kubernetes - exitOnError(controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay), "error setting up RayCluster webhook") - } + go setupControllers(mgr, kubeClient, cfg, OpenShift, certsReady) + + exitOnError(mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping), "unable to set up health check") + exitOnError(mgr.AddReadyzCheck(cfg.Health.ReadinessEndpointName, healthz.Ping), "unable to set up ready check") + + setupLog.Info("starting manager") + exitOnError(mgr.Start(ctx), "error running manager") +} + +func setupControllers(mgr ctrl.Manager, dc discovery.DiscoveryInterface, cfg *config.CodeFlareOperatorConfiguration, OpenShift bool, certsReady chan struct{}) { + setupLog.Info("Waiting for certificate generation to complete") + <-certsReady + setupLog.Info("Certs ready") - ok, err := hasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster")) + exitOnError(controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay), "error setting up RayCluster webhook") + + ok, err := hasAPIResourceForGVK(dc, rayv1.GroupVersion.WithKind("RayCluster")) if ok { rayClusterController := controllers.RayClusterReconciler{ Client: mgr.GetClient(), @@ -162,12 +183,37 @@ func main() { } else if err != nil { exitOnError(err, "Could not determine if RayCluster CR present on cluster.") } +} - exitOnError(mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping), "unable to set up health check") - exitOnError(mgr.AddReadyzCheck(cfg.Health.ReadinessEndpointName, healthz.Ping), "unable to set up ready check") +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update +// +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=mutatingwebhookconfigurations,verbs=get;list;watch;update +// +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=validatingwebhookconfigurations,verbs=get;list;watch;update - setupLog.Info("starting manager") - exitOnError(mgr.Start(ctx), "error running manager") +func setupCertManagement(mgr ctrl.Manager, namespace string, certsReady chan struct{}) error { + return cert.AddRotator(mgr, &cert.CertRotator{ + SecretKey: types.NamespacedName{ + Namespace: namespace, + Name: "codeflare-operator-webhook-server-cert", + }, + CertDir: "/tmp/k8s-webhook-server/serving-certs", + CAName: "codeflare", + CAOrganization: "openshift.ai", + DNSName: fmt.Sprintf("%s.%s.svc", "codeflare-operator-webhook-service", namespace), + IsReady: certsReady, + Webhooks: []cert.WebhookInfo{ + { + Type: cert.Validating, + Name: "codeflare-operator-validating-webhook-configuration", + }, + { + Type: cert.Mutating, + Name: "codeflare-operator-mutating-webhook-configuration", + }, + }, + // When the controller is running in the leader election mode, + // we expect webhook server will run in primary and secondary instance + RequireLeaderElection: false, + }) } func loadIntoOrCreate(ctx context.Context, client kubernetes.Interface, ns, name string, cfg *config.CodeFlareOperatorConfiguration) error { diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 27a4b56b..b4078119 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -24,7 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -66,7 +66,7 @@ var _ webhook.CustomValidator = &rayClusterWebhook{} func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error { rayCluster := obj.(*rayv1.RayCluster) - if !pointer.BoolDeref(w.Config.RayDashboardOAuthEnabled, true) { + if !ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) { return nil } @@ -143,7 +143,7 @@ func validateOAuthProxyVolume(rayCluster *rayv1.RayCluster) field.ErrorList { func validateIngress(rayCluster *rayv1.RayCluster) field.ErrorList { var allErrors field.ErrorList - if pointer.BoolDeref(rayCluster.Spec.HeadGroupSpec.EnableIngress, false) { + if ptr.Deref(rayCluster.Spec.HeadGroupSpec.EnableIngress, false) { allErrors = append(allErrors, field.Invalid( field.NewPath("spec", "headGroupSpec", "enableIngress"), rayCluster.Spec.HeadGroupSpec.EnableIngress, From ffc555aa51615e7a3468e4797f578b4f73918747 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 09:55:45 +0200 Subject: [PATCH 353/377] Skip OAuth proxy validation when it's disabled --- pkg/controllers/raycluster_webhook.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index b4078119..20ebe3fe 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -104,9 +104,12 @@ func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj r } allErrors = append(allErrors, validateIngress(rayCluster)...) - allErrors = append(allErrors, validateOAuthProxyContainer(rayCluster)...) - allErrors = append(allErrors, validateOAuthProxyVolume(rayCluster)...) - allErrors = append(allErrors, validateHeadGroupServiceAccountName(rayCluster)...) + + if ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) { + allErrors = append(allErrors, validateOAuthProxyContainer(rayCluster)...) + allErrors = append(allErrors, validateOAuthProxyVolume(rayCluster)...) + allErrors = append(allErrors, validateHeadGroupServiceAccountName(rayCluster)...) + } return warnings, allErrors.ToAggregate() } From cdf154a10b310859accf5508840dc6a68ddb83f4 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 10:13:48 +0200 Subject: [PATCH 354/377] Add missing validations on RayCluster creation --- pkg/controllers/raycluster_webhook.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 20ebe3fe..b29794ee 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -89,6 +89,12 @@ func (w *rayClusterWebhook) ValidateCreate(ctx context.Context, obj runtime.Obje allErrors = append(allErrors, validateIngress(rayCluster)...) + if ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) { + allErrors = append(allErrors, validateOAuthProxyContainer(rayCluster)...) + allErrors = append(allErrors, validateOAuthProxyVolume(rayCluster)...) + allErrors = append(allErrors, validateHeadGroupServiceAccountName(rayCluster)...) + } + return warnings, allErrors.ToAggregate() } From 2fe0a5257cbadb22210e20629e23eab1d774eb18 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 10:15:18 +0200 Subject: [PATCH 355/377] Delay webhook readiness until certicates are set up --- main.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 4228d1b4..700f12e8 100644 --- a/main.go +++ b/main.go @@ -18,8 +18,10 @@ package main import ( "context" + "errors" "flag" "fmt" + "net/http" "os" "strings" "time" @@ -153,18 +155,16 @@ func main() { certsReady := make(chan struct{}) exitOnError(setupCertManagement(mgr, namespace, certsReady), "unable to setup cert-controller") - OpenShift := isOpenShift(ctx, kubeClient.DiscoveryClient) + go setupControllers(mgr, kubeClient, cfg, isOpenShift(ctx, kubeClient.DiscoveryClient), certsReady) - go setupControllers(mgr, kubeClient, cfg, OpenShift, certsReady) - - exitOnError(mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping), "unable to set up health check") - exitOnError(mgr.AddReadyzCheck(cfg.Health.ReadinessEndpointName, healthz.Ping), "unable to set up ready check") + setupLog.Info("setting up health endpoints") + exitOnError(setupProbeEndpoints(mgr, cfg, certsReady), "unable to set up health check") setupLog.Info("starting manager") exitOnError(mgr.Start(ctx), "error running manager") } -func setupControllers(mgr ctrl.Manager, dc discovery.DiscoveryInterface, cfg *config.CodeFlareOperatorConfiguration, OpenShift bool, certsReady chan struct{}) { +func setupControllers(mgr ctrl.Manager, dc discovery.DiscoveryInterface, cfg *config.CodeFlareOperatorConfiguration, isOpenShift bool, certsReady chan struct{}) { setupLog.Info("Waiting for certificate generation to complete") <-certsReady setupLog.Info("Certs ready") @@ -177,7 +177,7 @@ func setupControllers(mgr ctrl.Manager, dc discovery.DiscoveryInterface, cfg *co Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg.KubeRay, - IsOpenShift: OpenShift, + IsOpenShift: isOpenShift, } exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller") } else if err != nil { @@ -216,6 +216,22 @@ func setupCertManagement(mgr ctrl.Manager, namespace string, certsReady chan str }) } +func setupProbeEndpoints(mgr ctrl.Manager, cfg *config.CodeFlareOperatorConfiguration, certsReady chan struct{}) error { + err := mgr.AddHealthzCheck(cfg.Health.LivenessEndpointName, healthz.Ping) + if err != nil { + return err + } + + return mgr.AddReadyzCheck(cfg.Health.ReadinessEndpointName, func(req *http.Request) error { + select { + case <-certsReady: + return mgr.GetWebhookServer().StartedChecker()(req) + default: + return errors.New("certificates are not ready") + } + }) +} + func loadIntoOrCreate(ctx context.Context, client kubernetes.Interface, ns, name string, cfg *config.CodeFlareOperatorConfiguration) error { configMap, err := client.CoreV1().ConfigMaps(ns).Get(ctx, name, metav1.GetOptions{}) if apierrors.IsNotFound(err) { From 64de673a8a69d9703d2e928fa0b1b6001344738f Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 11:14:14 +0200 Subject: [PATCH 356/377] Update OWNERS --- OWNERS | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/OWNERS b/OWNERS index c962aaaf..955704b3 100644 --- a/OWNERS +++ b/OWNERS @@ -1,20 +1,19 @@ approvers: - anishasthana - astefanutti + - ChristianZaccaria - jbusche - kpostoffice - - maxusmusti - sutaakar - tedhtchang reviewers: - anishasthana - astefanutti + - Bobbins228 - ChristianZaccaria - dimakis - Fiona-Waters - jbusche - kpostoffice - - maxusmusti - - MichaelClifford - sutaakar - tedhtchang From 9b99614c1ca72cb3200592362d2dda787f9677e2 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 16 Apr 2024 17:44:02 +0100 Subject: [PATCH 357/377] CVE fixes of High priority --- go.mod | 12 +- go.sum | 575 ++------------------------------------------------------- 2 files changed, 23 insertions(+), 564 deletions(-) diff --git a/go.mod b/go.mod index 06797c21..b49f2013 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,8 @@ require ( replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 +replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 + require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -58,22 +60,22 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect + github.com/openshift-online/ocm-sdk-go v0.1.411 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.8.4 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/net v0.20.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.16.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.12.0 // indirect diff --git a/go.sum b/go.sum index 8a496b7e..0b63f573 100644 --- a/go.sum +++ b/go.sum @@ -1,95 +1,27 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -101,227 +33,85 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= -github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= -github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= -github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= -github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= -github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= -github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= -github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= -github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= -github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= -github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.12.0 h1:/RvQ24k3TnNdfBSW0ou9EOi5jx2cX7zfE8n2nLKuiP0= -github.com/jackc/pgconn v1.12.0/go.mod h1:ZkhRC59Llhrq3oSfrikvwQ5NaxYExr6twkdkMLaKono= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= -github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= -github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= -github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= -github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL8Y= -github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= -github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= -github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= -github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs= -github.com/jackc/pgtype v1.11.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= -github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= -github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= -github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.16.0 h1:4k1tROTJctHotannFYzu77dY3bgtMRymQP7tXQjqpPk= -github.com/jackc/pgx/v4 v4.16.0/go.mod h1:N0A9sFdWzkw/Jy1lwoiB64F2+ugFZi987zRxcPez/wI= -github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5nbVfmstNpj4kHQFkvPK4= github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= -github.com/openshift-online/ocm-sdk-go v0.1.368 h1:qP+gkChV8WDwwpkUw1xUyjTXKdvrwyd70Gff2GMUSeU= -github.com/openshift-online/ocm-sdk-go v0.1.368/go.mod h1:KYOw8kAKAHyPrJcQoVR82CneQ4ofC02Na4cXXaTq4Nw= +github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= +github.com/openshift-online/ocm-sdk-go v0.1.411/go.mod h1:CiAu2jwl3ITKOxkeV0Qnhzv4gs35AmpIzVABQLtcI2Y= github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -331,457 +121,127 @@ github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a/go.mod h1:2Ck9LC+6Xi4jTDSlCJoP00tCzSrxek0roLsjvUgL2gY= github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 h1:oyhdLdc4BgA4zcH1zlRrSrYpzuVxV5QLDbyIXrwnQqs= github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= -github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= +github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= -golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg= k8s.io/apiextensions-apiserver v0.28.1 h1:l2ThkBRjrWpw4f24uq0Da2HaEgqJZ7pcgiEUTKSmQZw= @@ -799,9 +259,6 @@ k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 h1:nVKRi5eItf3x9kkIMfdT4D k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.16.1 h1:+15lzrmHsE0s2kNl0Dl8cTchI5Cs8qofo5PGcPrV9z0= sigs.k8s.io/controller-runtime v0.16.1/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= From 7caa6c750d464f448dade235a3f6c144af8a057c Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 16 Apr 2024 18:01:17 +0100 Subject: [PATCH 358/377] Replace pgx/v4 for pgx/v5 --- go.mod | 3 +++ go.sum | 11 +++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b49f2013..47e28213 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,8 @@ replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apise replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 +replace github.com/jackc/pgx/v4 => github.com/jackc/pgx/v5 v5.5.4 + require ( github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -53,6 +55,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect + github.com/jackc/pgx/v5 v5.5.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index 0b63f573..8d177bc0 100644 --- a/go.sum +++ b/go.sum @@ -66,14 +66,12 @@ github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= -github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= -github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= -github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= -github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= +github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -187,6 +185,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 07098c01cc3397573f19c5b8219397d0806d5a9d Mon Sep 17 00:00:00 2001 From: Kevin Postlethwait Date: Fri, 19 Apr 2024 10:20:10 -0400 Subject: [PATCH 359/377] add additional function which creates network policy (#498) * add additional function which creates network policy Signed-off-by: Kevin * Locate KubeRay deployment from DSCInitialization application namespace * tidy go mod files Signed-off-by: Kevin * err undefined Signed-off-by: Kevin * add dsci to schema instead of odh Signed-off-by: Kevin * add roles for creating nwp Signed-off-by: Kevin * add rule for metrics project Signed-off-by: Kevin * add missing dsci import Signed-off-by: Kevin --------- Signed-off-by: Kevin Co-authored-by: Antonin Stefanutti --- config/rbac/role.yaml | 18 ++ go.mod | 34 +-- go.sum | 253 +++++++++++++++++++---- main.go | 3 + pkg/controllers/raycluster_controller.go | 74 +++++++ 5 files changed, 329 insertions(+), 53 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 24ac7087..13709839 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -73,6 +73,14 @@ rules: - get - patch - update +- apiGroups: + - dscinitialization.opendatahub.io + resources: + - dscinitializations + verbs: + - get + - list + - watch - apiGroups: - networking.k8s.io resources: @@ -83,6 +91,16 @@ rules: - get - patch - update +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - patch + - update - apiGroups: - ray.io resources: diff --git a/go.mod b/go.mod index 47e28213..57c47c46 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,27 @@ module github.com/project-codeflare/codeflare-operator go 1.20 require ( - github.com/onsi/ginkgo/v2 v2.11.0 + github.com/onsi/ginkgo/v2 v2.12.1 github.com/onsi/gomega v1.27.10 github.com/open-policy-agent/cert-controller v0.10.1 - github.com/openshift/api v0.0.0-20230213134911-7ba313770556 + github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 + github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a github.com/ray-project/kuberay/ray-operator v1.0.0 go.uber.org/zap v1.26.0 - k8s.io/api v0.28.1 - k8s.io/apimachinery v0.28.1 - k8s.io/client-go v0.28.1 - k8s.io/component-base v0.28.1 + k8s.io/api v0.28.3 + k8s.io/apimachinery v0.28.3 + k8s.io/client-go v11.0.0+incompatible + k8s.io/component-base v0.28.3 k8s.io/klog/v2 v2.100.1 k8s.io/utils v0.0.0-20230726121419-3b25d923346b sigs.k8s.io/controller-runtime v0.16.1 sigs.k8s.io/yaml v1.3.0 ) +replace k8s.io/client-go => k8s.io/client-go v0.28.3 + replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 replace go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 @@ -37,8 +40,8 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.4 // indirect - github.com/go-logr/zapr v1.2.4 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -52,9 +55,9 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.0 // indirect - github.com/imdario/mergo v0.3.12 // indirect + github.com/imdario/mergo v0.3.13 // indirect github.com/jackc/pgx/v5 v5.5.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -64,6 +67,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/openshift-online/ocm-sdk-go v0.1.411 // indirect + github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 // indirect github.com/prometheus/client_golang v1.18.0 // indirect @@ -74,22 +78,22 @@ require ( github.com/spf13/pflag v1.0.5 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.12.0 // indirect + golang.org/x/tools v0.13.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.1 // indirect + k8s.io/apiextensions-apiserver v0.28.2 // indirect k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect ) diff --git a/go.sum b/go.sum index 8d177bc0..d67f55ff 100644 --- a/go.sum +++ b/go.sum @@ -1,69 +1,124 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= -github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -75,41 +130,69 @@ github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= +github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/open-policy-agent/cert-controller v0.10.1 h1:RXSYoyn8FdCenWecRP//UV5nbVfmstNpj4kHQFkvPK4= github.com/open-policy-agent/cert-controller v0.10.1/go.mod h1:4uRbBLY5DsPOog+a9pqk3JLxuuhrWsbUedQW65HcLTI= github.com/open-policy-agent/frameworks/constraint v0.0.0-20230822235116-f0b62fe1e4c4 h1:5dum5SLEz+95JDLkMls7Z7IDPjvSq3UhJSFe4f5einQ= +github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0 h1:tOX6R3N41pdyC+E1TeLErVY7KJV0zg9GAd/Z7xLT7no= +github.com/opendatahub-io/opendatahub-operator/v2 v2.10.0/go.mod h1:Hgy6bUPl29drwjs9F/5PZHUopOOojQpAPv1mWh3jnJQ= github.com/openshift-online/ocm-sdk-go v0.1.411 h1:DlNHC3yqmk77Wzc+YJBsd0ccHXn7JFwGC1C1NOp/faw= github.com/openshift-online/ocm-sdk-go v0.1.411/go.mod h1:CiAu2jwl3ITKOxkeV0Qnhzv4gs35AmpIzVABQLtcI2Y= -github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= -github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A= +github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 h1:e3zIxk67/kiABxGFfFVECqJ4FcQRG5DPF8lgDV9f+MM= +github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= +github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= +github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -121,6 +204,7 @@ github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0 h1:oyhdLdc4BgA github.com/project-codeflare/multi-cluster-app-dispatcher v1.37.0/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= @@ -129,15 +213,18 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -147,122 +234,212 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= -k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg= -k8s.io/apiextensions-apiserver v0.28.1 h1:l2ThkBRjrWpw4f24uq0Da2HaEgqJZ7pcgiEUTKSmQZw= -k8s.io/apiextensions-apiserver v0.28.1/go.mod h1:sVvrI+P4vxh2YBBcm8n2ThjNyzU4BQGilCQ/JAY5kGs= -k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= -k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= -k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8= -k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE= -k8s.io/component-base v0.28.1 h1:LA4AujMlK2mr0tZbQDZkjWbdhTV5bRyEyAFe0TJxlWg= -k8s.io/component-base v0.28.1/go.mod h1:jI11OyhbX21Qtbav7JkhehyBsIRfnO8oEgoAR12ArIU= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= +k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= +k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= +k8s.io/apiextensions-apiserver v0.28.2 h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU= +k8s.io/apiextensions-apiserver v0.28.2/go.mod h1:5tnkxLGa9nefefYzWuAlWZ7RZYuN/765Au8cWLA6SRg= +k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= +k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= +k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= +k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= +k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= +k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= +k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-aggregator v0.28.1 h1:rvG4llYnQKHjj6YjjoBPEJxfD1uH0DJwkrJTNKGAaCs= +k8s.io/kube-aggregator v0.28.3 h1:CVbj3+cpshSHR5dWPzLYx3sVpIDEPLlzMSxY/lAc9cM= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 h1:nVKRi5eItf3x9kkIMfdT4D1/LqPzj0bLjxLYWbdUtV0= k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.16.1 h1:+15lzrmHsE0s2kNl0Dl8cTchI5Cs8qofo5PGcPrV9z0= sigs.k8s.io/controller-runtime v0.16.1/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/main.go b/main.go index 700f12e8..134e7153 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,7 @@ import ( "time" cert "github.com/open-policy-agent/cert-controller/pkg/rotator" + dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" "go.uber.org/zap/zapcore" @@ -73,6 +74,8 @@ func init() { utilruntime.Must(rayv1.AddToScheme(scheme)) // OpenShift Route utilruntime.Must(routev1.Install(scheme)) + // ODH + utilruntime.Must(dsciv1.AddToScheme(scheme)) } func main() { diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index ba6ca84f..4b08b209 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -23,6 +23,7 @@ import ( "encoding/base64" "fmt" + dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" corev1 "k8s.io/api/core/v1" @@ -32,7 +33,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" coreapply "k8s.io/client-go/applyconfigurations/core/v1" + metav1apply "k8s.io/client-go/applyconfigurations/meta/v1" v1 "k8s.io/client-go/applyconfigurations/meta/v1" + networkingapply "k8s.io/client-go/applyconfigurations/networking/v1" rbacapply "k8s.io/client-go/applyconfigurations/rbac/v1" "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" @@ -83,6 +86,8 @@ var ( // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;create;update;patch;delete // +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create; // +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create; +// +kubebuilder:rbac:groups=dscinitialization.opendatahub.io,resources=dscinitializations,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;create;update;patch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -205,6 +210,25 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } + // Locate the KubeRay operator deployment: + // - First try to get the ODH / RHOAI application namespace from the DSCInitialization + // - Or fallback to the well-known defaults + var kubeRayNamespaces []string + dsci := &dsciv1.DSCInitialization{} + err := r.Client.Get(ctx, client.ObjectKey{Name: "default-dsci"}, dsci) + if errors.IsNotFound(err) { + kubeRayNamespaces = []string{"opendatahub", "redhat-ods-applications"} + } else if err != nil { + return ctrl.Result{}, err + } else { + kubeRayNamespaces = []string{dsci.Spec.ApplicationsNamespace} + } + + _, err = r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredNetworkPolicy(cluster, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to update NetworkPolicy") + } + return ctrl.Result{}, nil } @@ -337,6 +361,56 @@ func desiredOAuthSecret(cluster *rayv1.RayCluster, r *RayClusterReconciler) *cor // Create a Kubernetes secret to store the cookie secret } +func desiredNetworkPolicy(cluster *rayv1.RayCluster, kubeRayNamespaces []string) *networkingapply.NetworkPolicyApplyConfiguration { + return networkingapply.NetworkPolicy(cluster.Name, cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithSpec(networkingapply.NetworkPolicySpec(). + WithPodSelector(metav1apply.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})). + WithIngress( + networkingapply.NetworkPolicyIngressRule(). + WithPorts( + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(6379)), + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), + ).WithFrom( + networkingapply.NetworkPolicyPeer().WithPodSelector(metav1apply.LabelSelector()), + ), + networkingapply.NetworkPolicyIngressRule(). + WithFrom( + networkingapply.NetworkPolicyPeer().WithPodSelector(metav1apply.LabelSelector(). + WithMatchLabels(map[string]string{"app.kubernetes.io/component": "kuberay-operator"})). + WithNamespaceSelector(metav1apply.LabelSelector(). + WithMatchExpressions(metav1apply.LabelSelectorRequirement(). + WithKey(corev1.LabelMetadataName). + WithOperator(metav1.LabelSelectorOpIn). + WithValues(kubeRayNamespaces...)))). + WithPorts( + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), + ), + networkingapply.NetworkPolicyIngressRule(). + WithPorts( + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), + ). + WithFrom( + networkingapply.NetworkPolicyPeer().WithNamespaceSelector(metav1apply.LabelSelector(). + WithMatchExpressions(metav1apply.LabelSelectorRequirement(). + WithKey(corev1.LabelMetadataName). + WithOperator(metav1.LabelSelectorOpIn). + WithValues("openshift-monitoring"))), + ), + networkingapply.NetworkPolicyIngressRule(). + WithPorts( + networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), + ), + ), + ). + WithOwnerReferences( + v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + ) +} + // SetupWithManager sets up the controller with the Manager. func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { r.kubeClient = kubernetes.NewForConfigOrDie(mgr.GetConfig()) From e47db52752041f373f3a6536e4c0abacd257f080 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Wed, 17 Apr 2024 17:27:24 +0100 Subject: [PATCH 360/377] Added Mtls patch (cherry picked from commit de2de96fc88022df783b637ccb145d1d73ba66ff) --- config/rbac/role.yaml | 6 + main.go | 25 +++ pkg/config/config.go | 2 + pkg/controllers/raycluster_webhook.go | 217 +++++++++++++++++++++++++- pkg/controllers/support.go | 11 ++ 5 files changed, 254 insertions(+), 7 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 13709839..899e4915 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -44,6 +44,12 @@ rules: - subjectaccessreviews verbs: - create +- apiGroups: + - config.openshift.io + resources: + - ingresses + verbs: + - get - apiGroups: - "" resources: diff --git a/main.go b/main.go index 134e7153..95b0d590 100644 --- a/main.go +++ b/main.go @@ -53,6 +53,7 @@ import ( "sigs.k8s.io/yaml" routev1 "github.com/openshift/api/route/v1" + clientset "github.com/openshift/client-go/config/clientset/versioned" "github.com/project-codeflare/codeflare-operator/pkg/config" "github.com/project-codeflare/codeflare-operator/pkg/controllers" @@ -78,6 +79,8 @@ func init() { utilruntime.Must(dsciv1.AddToScheme(scheme)) } +// +kubebuilder:rbac:groups=config.openshift.io,resources=ingresses,verbs=get; + func main() { var configMapName string flag.StringVar(&configMapName, "config", "codeflare-operator-config", @@ -120,6 +123,7 @@ func main() { KubeRay: &config.KubeRayConfiguration{ RayDashboardOAuthEnabled: ptr.To(true), IngressDomain: "", + MTLSEnabled: ptr.To(true), }, } @@ -158,6 +162,13 @@ func main() { certsReady := make(chan struct{}) exitOnError(setupCertManagement(mgr, namespace, certsReady), "unable to setup cert-controller") + if cfg.KubeRay.IngressDomain == "" { + configClient, err := clientset.NewForConfig(kubeConfig) + exitOnError(err, "unable to create Route Client Set") + cfg.KubeRay.IngressDomain, err = getClusterDomain(ctx, configClient) + exitOnError(err, cfg.KubeRay.IngressDomain) + } + go setupControllers(mgr, kubeClient, cfg, isOpenShift(ctx, kubeClient.DiscoveryClient), certsReady) setupLog.Info("setting up health endpoints") @@ -335,3 +346,17 @@ func isOpenShift(ctx context.Context, dc discovery.DiscoveryInterface) bool { logger.Info("We detected being on Vanilla Kubernetes!") return false } + +func getClusterDomain(ctx context.Context, configClient *clientset.Clientset) (string, error) { + ingress, err := configClient.ConfigV1().Ingresses().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("failed to get Ingress object: %v", err) + } + + domain := ingress.Spec.Domain + if domain == "" { + return "", fmt.Errorf("domain is not set in the Ingress object") + } + + return domain, nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 08e2579b..af3bc349 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -35,6 +35,8 @@ type KubeRayConfiguration struct { RayDashboardOAuthEnabled *bool `json:"rayDashboardOAuthEnabled,omitempty"` IngressDomain string `json:"ingressDomain"` + + MTLSEnabled *bool `json:"mTLSEnabled,omitempty"` } type ControllerManager struct { diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index b29794ee..1e5b0b87 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "strconv" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -36,6 +37,7 @@ import ( const ( oauthProxyContainerName = "oauth-proxy" oauthProxyVolumeName = "proxy-tls-secret" + initContainerName = "create-cert" ) // log is for logging in this package. @@ -66,17 +68,44 @@ var _ webhook.CustomValidator = &rayClusterWebhook{} func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) error { rayCluster := obj.(*rayv1.RayCluster) - if !ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) { - return nil - } + if ptr.Deref(w.Config.RayDashboardOAuthEnabled, true) { + rayclusterlog.V(2).Info("Adding OAuth sidecar container") + rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, oauthProxyContainer(rayCluster), withContainerName(oauthProxyContainerName)) - rayclusterlog.V(2).Info("Adding OAuth sidecar container") + rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), withVolumeName(oauthProxyVolumeName)) - rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers, oauthProxyContainer(rayCluster), withContainerName(oauthProxyContainerName)) + rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy" + } - rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, oauthProxyTLSSecretVolume(rayCluster), withVolumeName(oauthProxyVolumeName)) + if ptr.Deref(w.Config.MTLSEnabled, true) { + rayclusterlog.V(2).Info("Adding create-cert Init Containers") + // HeadGroupSpec // + // Append the list of environment variables for the ray-head container + for _, envVar := range envVarList() { + rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, envVar, withEnvVarName(envVar.Name)) + } + + // Append the create-cert Init Container + rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, w.Config.IngressDomain), withContainerName(initContainerName)) + + // Append the CA volumes + for _, caVol := range caVolumes(rayCluster) { + rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, caVol, withVolumeName(caVol.Name)) + } + // WorkerGroupSpec // + // Append the list of environment variables for the worker container + for _, envVar := range envVarList() { + rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env, envVar, withEnvVarName(envVar.Name)) + } + + // Append the CA volumes + for _, caVol := range caVolumes(rayCluster) { + rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVol, withVolumeName(caVol.Name)) + } + // Append the create-cert Init Container + rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(), withContainerName(initContainerName)) - rayCluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = rayCluster.Name + "-oauth-proxy" + } return nil } @@ -117,6 +146,14 @@ func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj r allErrors = append(allErrors, validateHeadGroupServiceAccountName(rayCluster)...) } + // Init Container related errors + if ptr.Deref(w.Config.MTLSEnabled, true) { + allErrors = append(allErrors, validateHeadInitContainer(rayCluster, w)...) + allErrors = append(allErrors, validateWorkerInitContainer(rayCluster)...) + allErrors = append(allErrors, validateHeadEnvVars(rayCluster)...) + allErrors = append(allErrors, validateWorkerEnvVars(rayCluster)...) + allErrors = append(allErrors, validateCaVolumes(rayCluster)...) + } return warnings, allErrors.ToAggregate() } @@ -225,3 +262,169 @@ func oauthProxyTLSSecretVolume(rayCluster *rayv1.RayCluster) corev1.Volume { }, } } + +func initCaVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: "ca-vol", + MountPath: "/home/ray/workspace/ca", + ReadOnly: true, + }, + { + Name: "server-cert", + MountPath: "/home/ray/workspace/tls", + ReadOnly: false, + }, + } +} + +func envVarList() []corev1.EnvVar { + return []corev1.EnvVar{ + { + Name: "MY_POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.podIP", + }, + }, + }, + { + Name: "RAY_USE_TLS", + Value: "1", + }, + { + Name: "RAY_TLS_SERVER_CERT", + Value: "/home/ray/workspace/tls/server.crt", + }, + { + Name: "RAY_TLS_SERVER_KEY", + Value: "/home/ray/workspace/tls/server.key", + }, + { + Name: "RAY_TLS_CA_CERT", + Value: "/home/ray/workspace/tls/ca.crt", + }, + } +} + +func caVolumes(rayCluster *rayv1.RayCluster) []corev1.Volume { + return []corev1.Volume{ + { + Name: "ca-vol", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: `ca-secret-` + rayCluster.Name, + }, + }, + }, + { + Name: "server-cert", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } +} + +func rayHeadInitContainer(rayCluster *rayv1.RayCluster, domain string) corev1.Container { + rayClientRoute := "rayclient-" + rayCluster.Name + "-" + rayCluster.Namespace + "." + domain + // Service name for basic interactive + svcDomain := rayCluster.Name + "-head-svc." + rayCluster.Namespace + ".svc" + + initContainerHead := corev1.Container{ + Name: "create-cert", + Image: "quay.io/project-codeflare/ray:latest-py39-cu118", + Command: []string{ + "sh", + "-c", + `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, + }, + VolumeMounts: initCaVolumeMounts(), + } + return initContainerHead +} + +func rayWorkerInitContainer() corev1.Container { + initContainerWorker := corev1.Container{ + Name: "create-cert", + Image: "quay.io/project-codeflare/ray:latest-py39-cu118", + Command: []string{ + "sh", + "-c", + `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, + }, + VolumeMounts: initCaVolumeMounts(), + } + return initContainerWorker +} + +func validateHeadInitContainer(rayCluster *rayv1.RayCluster, w *rayClusterWebhook) field.ErrorList { + var allErrors field.ErrorList + + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, w.Config.IngressDomain), byContainerName, + field.NewPath("spec", "headGroupSpec", "template", "spec", "initContainers"), + "create-cert Init Container is immutable"); err != nil { + allErrors = append(allErrors, err) + } + + return allErrors +} + +func validateWorkerInitContainer(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(), byContainerName, + field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "initContainers"), + "create-cert Init Container is immutable"); err != nil { + allErrors = append(allErrors, err) + } + + return allErrors +} + +func validateCaVolumes(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + for _, caVol := range caVolumes(rayCluster) { + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, caVol, byVolumeName, + field.NewPath("spec", "headGroupSpec", "template", "spec", "volumes"), + "ca-vol and server-cert Secret volumes are immutable"); err != nil { + allErrors = append(allErrors, err) + } + if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVol, byVolumeName, + field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "volumes"), + "ca-vol and server-cert Secret volumes are immutable"); err != nil { + allErrors = append(allErrors, err) + } + } + + return allErrors +} + +func validateHeadEnvVars(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + for _, envVar := range envVarList() { + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, envVar, byEnvVarName, + field.NewPath("spec", "headGroupSpec", "template", "spec", "containers", strconv.Itoa(0), "env"), + "RAY_TLS related environment variables are immutable"); err != nil { + allErrors = append(allErrors, err) + } + } + + return allErrors +} + +func validateWorkerEnvVars(rayCluster *rayv1.RayCluster) field.ErrorList { + var allErrors field.ErrorList + + for _, envVar := range envVarList() { + if err := contains(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env, envVar, byEnvVarName, + field.NewPath("spec", "workerGroupSpecs", "0", "template", "spec", "containers", strconv.Itoa(0), "env"), + "RAY_TLS related environment variables are immutable"); err != nil { + allErrors = append(allErrors, err) + } + } + + return allErrors +} diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 0efbc747..0ea8f424 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -140,3 +140,14 @@ func withVolumeName(name string) compare[corev1.Volume] { return v1.Name == name } } + +var byEnvVarName = compare[corev1.EnvVar]( + func(e1, e2 corev1.EnvVar) bool { + return e1.Name == e2.Name + }) + +func withEnvVarName(name string) compare[corev1.EnvVar] { + return func(e1, e2 corev1.EnvVar) bool { + return e1.Name == name + } +} From f5be3340d7fde1e4ebdf96b527607d94cd47efb9 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Fri, 19 Apr 2024 11:24:02 +0100 Subject: [PATCH 361/377] Added Ingress domain to validateHeadInitContainer --- pkg/controllers/raycluster_webhook.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 1e5b0b87..9362afe5 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -148,7 +148,7 @@ func (w *rayClusterWebhook) ValidateUpdate(ctx context.Context, oldObj, newObj r // Init Container related errors if ptr.Deref(w.Config.MTLSEnabled, true) { - allErrors = append(allErrors, validateHeadInitContainer(rayCluster, w)...) + allErrors = append(allErrors, validateHeadInitContainer(rayCluster, w.Config.IngressDomain)...) allErrors = append(allErrors, validateWorkerInitContainer(rayCluster)...) allErrors = append(allErrors, validateHeadEnvVars(rayCluster)...) allErrors = append(allErrors, validateWorkerEnvVars(rayCluster)...) @@ -358,10 +358,10 @@ func rayWorkerInitContainer() corev1.Container { return initContainerWorker } -func validateHeadInitContainer(rayCluster *rayv1.RayCluster, w *rayClusterWebhook) field.ErrorList { +func validateHeadInitContainer(rayCluster *rayv1.RayCluster, domain string) field.ErrorList { var allErrors field.ErrorList - if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, w.Config.IngressDomain), byContainerName, + if err := contains(rayCluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, rayHeadInitContainer(rayCluster, domain), byContainerName, field.NewPath("spec", "headGroupSpec", "template", "spec", "initContainers"), "create-cert Init Container is immutable"); err != nil { allErrors = append(allErrors, err) From 0bf2a4fa1c17b081400a25ff0dc115d69d79d673 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 14:48:58 +0200 Subject: [PATCH 362/377] Generate RayCluster CA certificate Secret --- main.go | 2 +- pkg/controllers/raycluster_controller.go | 197 ++++++++++++++++------- 2 files changed, 140 insertions(+), 59 deletions(-) diff --git a/main.go b/main.go index 95b0d590..31afbcbb 100644 --- a/main.go +++ b/main.go @@ -79,7 +79,7 @@ func init() { utilruntime.Must(dsciv1.AddToScheme(scheme)) } -// +kubebuilder:rbac:groups=config.openshift.io,resources=ingresses,verbs=get; +// +kubebuilder:rbac:groups=config.openshift.io,resources=ingresses,verbs=get func main() { var configMapName string diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 4b08b209..b3028755 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -19,9 +19,16 @@ package controllers import ( "context" "crypto/rand" + "crypto/rsa" "crypto/sha1" + "crypto/x509" + "crypto/x509/pkix" "encoding/base64" + "encoding/pem" "fmt" + "math/big" + rand2 "math/rand" + "time" dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" @@ -32,18 +39,18 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" - coreapply "k8s.io/client-go/applyconfigurations/core/v1" - metav1apply "k8s.io/client-go/applyconfigurations/meta/v1" - v1 "k8s.io/client-go/applyconfigurations/meta/v1" - networkingapply "k8s.io/client-go/applyconfigurations/networking/v1" - rbacapply "k8s.io/client-go/applyconfigurations/rbac/v1" + corev1ac "k8s.io/client-go/applyconfigurations/core/v1" + metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" + networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1" + rbacv1ac "k8s.io/client-go/applyconfigurations/rbac/v1" "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" routev1 "github.com/openshift/api/route/v1" - routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1" + routev1ac "github.com/openshift/client-go/route/applyconfigurations/route/v1" routev1client "github.com/openshift/client-go/route/clientset/versioned/typed/route/v1" "github.com/project-codeflare/codeflare-operator/pkg/config" @@ -144,6 +151,26 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } + if isMTLSEnabled(r.Config) { + caSecretName := caSecretNameFromCluster(cluster) + _, err := r.kubeClient.CoreV1().Secrets(cluster.Namespace).Get(ctx, caSecretName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + key, cert, err := generateCACertificate() + if err != nil { + logger.Error(err, "Failed to generate CA certificate") + return ctrl.Result{RequeueAfter: requeueTime}, err + } + _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredCASecret(cluster, key, cert), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to create CA Secret") + return ctrl.Result{RequeueAfter: requeueTime}, err + } + } else if err != nil { + logger.Error(err, "Failed to get CA Secret") + return ctrl.Result{RequeueAfter: requeueTime}, err + } + } + if cluster.Status.State != "suspended" && isRayDashboardOAuthEnabled(r.Config) && r.IsOpenShift { logger.Info("Creating OAuth Objects") _, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) @@ -152,7 +179,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{RequeueAfter: requeueTime}, err } - _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(cluster, r.CookieSalt), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to create OAuth Secret") return ctrl.Result{RequeueAfter: requeueTime}, err @@ -244,28 +271,29 @@ func getIngressHost(cfg *config.KubeRayConfiguration, cluster *rayv1.RayCluster, } func isRayDashboardOAuthEnabled(cfg *config.KubeRayConfiguration) bool { - if cfg != nil && cfg.RayDashboardOAuthEnabled != nil { - return *cfg.RayDashboardOAuthEnabled - } - return true + return cfg == nil || ptr.Deref(cfg.RayDashboardOAuthEnabled, true) +} + +func isMTLSEnabled(cfg *config.KubeRayConfiguration) bool { + return cfg == nil || ptr.Deref(cfg.MTLSEnabled, true) } func crbNameFromCluster(cluster *rayv1.RayCluster) string { return cluster.Name + "-" + cluster.Namespace + "-auth" // NOTE: potential naming conflicts ie {name: foo, ns: bar-baz} and {name: foo-bar, ns: baz} } -func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacapply.ClusterRoleBindingApplyConfiguration { - return rbacapply.ClusterRoleBinding( +func desiredOAuthClusterRoleBinding(cluster *rayv1.RayCluster) *rbacv1ac.ClusterRoleBindingApplyConfiguration { + return rbacv1ac.ClusterRoleBinding( crbNameFromCluster(cluster)). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithSubjects( - rbacapply.Subject(). + rbacv1ac.Subject(). WithKind("ServiceAccount"). WithName(oauthServiceAccountNameFromCluster(cluster)). WithNamespace(cluster.Namespace), ). WithRoleRef( - rbacapply.RoleRef(). + rbacv1ac.RoleRef(). WithAPIGroup("rbac.authorization.k8s.io"). WithKind("ClusterRole"). WithName("system:auth-delegator"), @@ -276,8 +304,8 @@ func oauthServiceAccountNameFromCluster(cluster *rayv1.RayCluster) string { return cluster.Name + "-oauth-proxy" } -func desiredServiceAccount(cluster *rayv1.RayCluster) *coreapply.ServiceAccountApplyConfiguration { - return coreapply.ServiceAccount(oauthServiceAccountNameFromCluster(cluster), cluster.Namespace). +func desiredServiceAccount(cluster *rayv1.RayCluster) *corev1ac.ServiceAccountApplyConfiguration { + return corev1ac.ServiceAccount(oauthServiceAccountNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithAnnotations(map[string]string{ "serviceaccounts.openshift.io/oauth-redirectreference.first": "" + @@ -285,7 +313,7 @@ func desiredServiceAccount(cluster *rayv1.RayCluster) *coreapply.ServiceAccountA `"reference":{"kind":"Route","name":"` + dashboardNameFromCluster(cluster) + `"}}`, }). WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) } @@ -297,19 +325,19 @@ func rayClientNameFromCluster(cluster *rayv1.RayCluster) string { return "rayclient-" + cluster.Name } -func desiredClusterRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration { - return routeapply.Route(dashboardNameFromCluster(cluster), cluster.Namespace). +func desiredClusterRoute(cluster *rayv1.RayCluster) *routev1ac.RouteApplyConfiguration { + return routev1ac.Route(dashboardNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). - WithSpec(routeapply.RouteSpec(). - WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(oauthServiceNameFromCluster(cluster))). - WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString((oAuthServicePortName)))). - WithTLS(routeapply.TLSConfig(). + WithSpec(routev1ac.RouteSpec(). + WithTo(routev1ac.RouteTargetReference().WithKind("Service").WithName(oauthServiceNameFromCluster(cluster))). + WithPort(routev1ac.RoutePort().WithTargetPort(intstr.FromString((oAuthServicePortName)))). + WithTLS(routev1ac.TLSConfig(). WithInsecureEdgeTerminationPolicy(routev1.InsecureEdgeTerminationPolicyRedirect). WithTermination(routev1.TLSTerminationReencrypt), ), ). WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) } @@ -321,14 +349,14 @@ func oauthServiceTLSSecretName(cluster *rayv1.RayCluster) string { return cluster.Name + "-proxy-tls-secret" } -func desiredOAuthService(cluster *rayv1.RayCluster) *coreapply.ServiceApplyConfiguration { - return coreapply.Service(oauthServiceNameFromCluster(cluster), cluster.Namespace). +func desiredOAuthService(cluster *rayv1.RayCluster) *corev1ac.ServiceApplyConfiguration { + return corev1ac.Service(oauthServiceNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithAnnotations(map[string]string{"service.beta.openshift.io/serving-cert-secret-name": oauthServiceTLSSecretName(cluster)}). WithSpec( - coreapply.ServiceSpec(). + corev1ac.ServiceSpec(). WithPorts( - coreapply.ServicePort(). + corev1ac.ServicePort(). WithName(oAuthServicePortName). WithPort(oAuthServicePort). WithTargetPort(intstr.FromString(oAuthServicePortName)). @@ -337,7 +365,7 @@ func desiredOAuthService(cluster *rayv1.RayCluster) *coreapply.ServiceApplyConfi WithSelector(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"}), ). WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) } @@ -346,68 +374,121 @@ func oauthSecretNameFromCluster(cluster *rayv1.RayCluster) string { } // desiredOAuthSecret defines the desired OAuth secret object -func desiredOAuthSecret(cluster *rayv1.RayCluster, r *RayClusterReconciler) *coreapply.SecretApplyConfiguration { +func desiredOAuthSecret(cluster *rayv1.RayCluster, cookieSalt string) *corev1ac.SecretApplyConfiguration { // Generate the cookie secret for the OAuth proxy hasher := sha1.New() // REVIEW is SHA1 okay here? - hasher.Write([]byte(cluster.Name + r.CookieSalt)) + hasher.Write([]byte(cluster.Name + cookieSalt)) cookieSecret := base64.StdEncoding.EncodeToString(hasher.Sum(nil)) - return coreapply.Secret(oauthSecretNameFromCluster(cluster), cluster.Namespace). + return corev1ac.Secret(oauthSecretNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithStringData(map[string]string{"cookie_secret": cookieSecret}). WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) - // Create a Kubernetes secret to store the cookie secret } -func desiredNetworkPolicy(cluster *rayv1.RayCluster, kubeRayNamespaces []string) *networkingapply.NetworkPolicyApplyConfiguration { - return networkingapply.NetworkPolicy(cluster.Name, cluster.Namespace). +func caSecretNameFromCluster(cluster *rayv1.RayCluster) string { + return "ca-secret-" + cluster.Name +} + +func desiredCASecret(cluster *rayv1.RayCluster, key, cert []byte) *corev1ac.SecretApplyConfiguration { + return corev1ac.Secret(caSecretNameFromCluster(cluster), cluster.Namespace). + WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). + WithData(map[string][]byte{ + corev1.TLSPrivateKeyKey: key, + corev1.TLSCertKey: cert, + }). + WithOwnerReferences(metav1ac.OwnerReference(). + WithUID(cluster.UID). + WithName(cluster.Name). + WithKind(cluster.Kind). + WithAPIVersion(cluster.APIVersion)) +} + +func generateCACertificate() ([]byte, []byte, error) { + serialNumber := big.NewInt(rand2.Int63()) + cert := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"OpenShift AI"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + certPrivateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + + privateKeyBytes := x509.MarshalPKCS1PrivateKey(certPrivateKey) + privateKeyPem := pem.EncodeToMemory( + &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: privateKeyBytes, + }, + ) + certBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &certPrivateKey.PublicKey, certPrivateKey) + certPem := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + + return privateKeyPem, certPem, nil +} + +func desiredNetworkPolicy(cluster *rayv1.RayCluster, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration { + return networkingv1ac.NetworkPolicy(cluster.Name, cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). - WithSpec(networkingapply.NetworkPolicySpec(). - WithPodSelector(metav1apply.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})). + WithSpec(networkingv1ac.NetworkPolicySpec(). + WithPodSelector(metav1ac.LabelSelector().WithMatchLabels(map[string]string{"ray.io/cluster": cluster.Name, "ray.io/node-type": "head"})). WithIngress( - networkingapply.NetworkPolicyIngressRule(). + networkingv1ac.NetworkPolicyIngressRule(). WithPorts( - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(6379)), - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(6379)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), ).WithFrom( - networkingapply.NetworkPolicyPeer().WithPodSelector(metav1apply.LabelSelector()), + networkingv1ac.NetworkPolicyPeer().WithPodSelector(metav1ac.LabelSelector()), ), - networkingapply.NetworkPolicyIngressRule(). + networkingv1ac.NetworkPolicyIngressRule(). WithFrom( - networkingapply.NetworkPolicyPeer().WithPodSelector(metav1apply.LabelSelector(). + networkingv1ac.NetworkPolicyPeer().WithPodSelector(metav1ac.LabelSelector(). WithMatchLabels(map[string]string{"app.kubernetes.io/component": "kuberay-operator"})). - WithNamespaceSelector(metav1apply.LabelSelector(). - WithMatchExpressions(metav1apply.LabelSelectorRequirement(). + WithNamespaceSelector(metav1ac.LabelSelector(). + WithMatchExpressions(metav1ac.LabelSelectorRequirement(). WithKey(corev1.LabelMetadataName). WithOperator(metav1.LabelSelectorOpIn). WithValues(kubeRayNamespaces...)))). WithPorts( - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8265)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), ), - networkingapply.NetworkPolicyIngressRule(). + networkingv1ac.NetworkPolicyIngressRule(). WithPorts( - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8080)), ). WithFrom( - networkingapply.NetworkPolicyPeer().WithNamespaceSelector(metav1apply.LabelSelector(). - WithMatchExpressions(metav1apply.LabelSelectorRequirement(). + networkingv1ac.NetworkPolicyPeer().WithNamespaceSelector(metav1ac.LabelSelector(). + WithMatchExpressions(metav1ac.LabelSelectorRequirement(). WithKey(corev1.LabelMetadataName). WithOperator(metav1.LabelSelectorOpIn). WithValues("openshift-monitoring"))), ), - networkingapply.NetworkPolicyIngressRule(). + networkingv1ac.NetworkPolicyIngressRule(). WithPorts( - networkingapply.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), ), ), ). WithOwnerReferences( - v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), + metav1ac.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion), ) } From 70a72b95f68d6999bea4c190caf9aeaa57079e4b Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 15:13:12 +0200 Subject: [PATCH 363/377] Update RayCluster CA Secret if it exists --- pkg/controllers/raycluster_controller.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index b3028755..c7ec3b3e 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -153,7 +153,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) if isMTLSEnabled(r.Config) { caSecretName := caSecretNameFromCluster(cluster) - _, err := r.kubeClient.CoreV1().Secrets(cluster.Namespace).Get(ctx, caSecretName, metav1.GetOptions{}) + caSecret, err := r.kubeClient.CoreV1().Secrets(cluster.Namespace).Get(ctx, caSecretName, metav1.GetOptions{}) if errors.IsNotFound(err) { key, cert, err := generateCACertificate() if err != nil { @@ -162,12 +162,20 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) } _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredCASecret(cluster, key, cert), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { - logger.Error(err, "Failed to create CA Secret") + logger.Error(err, "Failed to apply CA Secret") return ctrl.Result{RequeueAfter: requeueTime}, err } } else if err != nil { logger.Error(err, "Failed to get CA Secret") return ctrl.Result{RequeueAfter: requeueTime}, err + } else { + key := caSecret.Data[corev1.TLSPrivateKeyKey] + cert := caSecret.Data[corev1.TLSCertKey] + _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredCASecret(cluster, key, cert), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + if err != nil { + logger.Error(err, "Failed to apply CA Secret") + return ctrl.Result{RequeueAfter: requeueTime}, err + } } } @@ -434,6 +442,10 @@ func generateCACertificate() ([]byte, []byte, error) { }, ) certBytes, err := x509.CreateCertificate(rand.Reader, cert, cert, &certPrivateKey.PublicKey, certPrivateKey) + if err != nil { + return nil, nil, err + } + certPem := pem.EncodeToMemory(&pem.Block{ Type: "CERTIFICATE", Bytes: certBytes, From 46d4bab81f04cd14f0b210b080fa8310e25be8ee Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 15:17:02 +0200 Subject: [PATCH 364/377] Fix RayCluster CA Secret keys --- pkg/controllers/raycluster_controller.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index c7ec3b3e..a3033c84 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -75,6 +75,9 @@ const ( oAuthServicePortName = "oauth-proxy" ingressServicePortName = "dashboard" logRequeueing = "requeueing" + + CAPrivateKeyKey = "ca.key" + CACertKey = "ca.crt" ) var ( @@ -169,8 +172,8 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Error(err, "Failed to get CA Secret") return ctrl.Result{RequeueAfter: requeueTime}, err } else { - key := caSecret.Data[corev1.TLSPrivateKeyKey] - cert := caSecret.Data[corev1.TLSCertKey] + key := caSecret.Data[CAPrivateKeyKey] + cert := caSecret.Data[CACertKey] _, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredCASecret(cluster, key, cert), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to apply CA Secret") @@ -404,8 +407,8 @@ func desiredCASecret(cluster *rayv1.RayCluster, key, cert []byte) *corev1ac.Secr return corev1ac.Secret(caSecretNameFromCluster(cluster), cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithData(map[string][]byte{ - corev1.TLSPrivateKeyKey: key, - corev1.TLSCertKey: cert, + CAPrivateKeyKey: key, + CACertKey: cert, }). WithOwnerReferences(metav1ac.OwnerReference(). WithUID(cluster.UID). From 0f7519806273774dfd6b5e3b4f402f42c8812aff Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 16:33:05 +0200 Subject: [PATCH 365/377] Update RayCluster CA certificate common and issuer names --- pkg/controllers/raycluster_controller.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index a3033c84..a60cda5e 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -422,7 +422,10 @@ func generateCACertificate() ([]byte, []byte, error) { cert := &x509.Certificate{ SerialNumber: serialNumber, Subject: pkix.Name{ - Organization: []string{"OpenShift AI"}, + CommonName: "root-ca", + }, + Issuer: pkix.Name{ + CommonName: "root-ca", }, NotBefore: time.Now(), NotAfter: time.Now().AddDate(1, 0, 0), From 1a0141e5d0db20597800f5326949d9ab488f4108 Mon Sep 17 00:00:00 2001 From: Antonin Stefanutti Date: Fri, 19 Apr 2024 16:34:05 +0200 Subject: [PATCH 366/377] Fix missing certificate volume mounts to RayCluster --- pkg/controllers/raycluster_webhook.go | 25 +++++++++++++++++++------ pkg/controllers/support.go | 5 +++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/pkg/controllers/raycluster_webhook.go b/pkg/controllers/raycluster_webhook.go index 9362afe5..eb530697 100644 --- a/pkg/controllers/raycluster_webhook.go +++ b/pkg/controllers/raycluster_webhook.go @@ -79,7 +79,8 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err if ptr.Deref(w.Config.MTLSEnabled, true) { rayclusterlog.V(2).Info("Adding create-cert Init Containers") - // HeadGroupSpec // + // HeadGroupSpec + // Append the list of environment variables for the ray-head container for _, envVar := range envVarList() { rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].Env, envVar, withEnvVarName(envVar.Name)) @@ -92,7 +93,14 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err for _, caVol := range caVolumes(rayCluster) { rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Volumes, caVol, withVolumeName(caVol.Name)) } - // WorkerGroupSpec // + + // Append the certificate volume mounts + for _, mount := range certVolumeMounts() { + rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].VolumeMounts = upsert(rayCluster.Spec.HeadGroupSpec.Template.Spec.Containers[0].VolumeMounts, mount, byVolumeMountName) + } + + // WorkerGroupSpec + // Append the list of environment variables for the worker container for _, envVar := range envVarList() { rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].Env, envVar, withEnvVarName(envVar.Name)) @@ -102,9 +110,14 @@ func (w *rayClusterWebhook) Default(ctx context.Context, obj runtime.Object) err for _, caVol := range caVolumes(rayCluster) { rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVol, withVolumeName(caVol.Name)) } + + // Append the certificate volume mounts + for _, mount := range certVolumeMounts() { + rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].VolumeMounts = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[0].VolumeMounts, mount, byVolumeMountName) + } + // Append the create-cert Init Container rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = upsert(rayCluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, rayWorkerInitContainer(), withContainerName(initContainerName)) - } return nil @@ -263,7 +276,7 @@ func oauthProxyTLSSecretVolume(rayCluster *rayv1.RayCluster) corev1.Volume { } } -func initCaVolumeMounts() []corev1.VolumeMount { +func certVolumeMounts() []corev1.VolumeMount { return []corev1.VolumeMount{ { Name: "ca-vol", @@ -339,7 +352,7 @@ func rayHeadInitContainer(rayCluster *rayv1.RayCluster, domain string) corev1.Co "-c", `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, }, - VolumeMounts: initCaVolumeMounts(), + VolumeMounts: certVolumeMounts(), } return initContainerHead } @@ -353,7 +366,7 @@ func rayWorkerInitContainer() corev1.Container { "-c", `cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`, }, - VolumeMounts: initCaVolumeMounts(), + VolumeMounts: certVolumeMounts(), } return initContainerWorker } diff --git a/pkg/controllers/support.go b/pkg/controllers/support.go index 0ea8f424..d208b52d 100644 --- a/pkg/controllers/support.go +++ b/pkg/controllers/support.go @@ -141,6 +141,11 @@ func withVolumeName(name string) compare[corev1.Volume] { } } +var byVolumeMountName = compare[corev1.VolumeMount]( + func(v1, v2 corev1.VolumeMount) bool { + return v1.Name == v2.Name + }) + var byEnvVarName = compare[corev1.EnvVar]( func(e1, e2 corev1.EnvVar) bool { return e1.Name == e2.Name From 1c8a64d78b48458aed7a01d989e1aa35bc50c5ff Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 19 Apr 2024 11:02:57 -0400 Subject: [PATCH 367/377] add entry for rayclient to allow all connections Signed-off-by: Kevin --- pkg/controllers/raycluster_controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index a60cda5e..6377e27b 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -502,6 +502,7 @@ func desiredNetworkPolicy(cluster *rayv1.RayCluster, kubeRayNamespaces []string) networkingv1ac.NetworkPolicyIngressRule(). WithPorts( networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), ), ), ). From 0c2aa476226627af731ebc584b667e0868bd00df Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 19 Apr 2024 11:34:16 -0400 Subject: [PATCH 368/377] add cfg check for mtls before enabling access to client Signed-off-by: Kevin --- pkg/controllers/raycluster_controller.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/raycluster_controller.go b/pkg/controllers/raycluster_controller.go index 6377e27b..41464af2 100644 --- a/pkg/controllers/raycluster_controller.go +++ b/pkg/controllers/raycluster_controller.go @@ -262,7 +262,7 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) kubeRayNamespaces = []string{dsci.Spec.ApplicationsNamespace} } - _, err = r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredNetworkPolicy(cluster, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) + _, err = r.kubeClient.NetworkingV1().NetworkPolicies(cluster.Namespace).Apply(ctx, desiredNetworkPolicy(cluster, r.Config, kubeRayNamespaces), metav1.ApplyOptions{FieldManager: controllerName, Force: true}) if err != nil { logger.Error(err, "Failed to update NetworkPolicy") } @@ -460,7 +460,13 @@ func generateCACertificate() ([]byte, []byte, error) { return privateKeyPem, certPem, nil } -func desiredNetworkPolicy(cluster *rayv1.RayCluster, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration { +func desiredNetworkPolicy(cluster *rayv1.RayCluster, cfg *config.KubeRayConfiguration, kubeRayNamespaces []string) *networkingv1ac.NetworkPolicyApplyConfiguration { + allSecuredPorts := []*networkingv1ac.NetworkPolicyPortApplyConfiguration{ + networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), + } + if ptr.Deref(cfg.MTLSEnabled, true) { + allSecuredPorts = append(allSecuredPorts, networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001))) + } return networkingv1ac.NetworkPolicy(cluster.Name, cluster.Namespace). WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}). WithSpec(networkingv1ac.NetworkPolicySpec(). @@ -501,8 +507,7 @@ func desiredNetworkPolicy(cluster *rayv1.RayCluster, kubeRayNamespaces []string) ), networkingv1ac.NetworkPolicyIngressRule(). WithPorts( - networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(8443)), - networkingv1ac.NetworkPolicyPort().WithProtocol(corev1.ProtocolTCP).WithPort(intstr.FromInt(10001)), + allSecuredPorts..., ), ), ). From fa700faab260065328d571e5fdf4c3e60d58eeb8 Mon Sep 17 00:00:00 2001 From: codeflare-machine-account <138894154+codeflare-machine-account@users.noreply.github.com> Date: Fri, 19 Apr 2024 18:28:32 +0200 Subject: [PATCH 369/377] Update dependency versions for release v1.4.0 (#540) Co-authored-by: codeflare-machine-account --- Makefile | 2 +- README.md | 6 +++--- config/manager/params.env | 2 +- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index cb8cd533..99e63324 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION ?= v0.0.0-dev BUNDLE_VERSION ?= $(VERSION:v%=%) # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing) -KUBERAY_VERSION ?= v1.0.0 +KUBERAY_VERSION ?= v1.1.0 # RAY_VERSION defines the default version of Ray (used for testing) RAY_VERSION ?= 2.5.0 diff --git a/README.md b/README.md index 423fb974..7531cfdd 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ CodeFlare Stack Compatibility Matrix | Component | Version | |------------------------------|---------------------------------------------------------------------------------------------------| -| CodeFlare Operator | [v1.3.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.3.1) | -| CodeFlare-SDK | [v0.15.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.15.1) | -| KubeRay | [v1.0.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.0.0) | +| CodeFlare Operator | [v1.4.0](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.0) | +| CodeFlare-SDK | [v0.16.0](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.0) | +| KubeRay | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0) | ## Development diff --git a/config/manager/params.env b/config/manager/params.env index 8843b5e3..6ac22e10 100644 --- a/config/manager/params.env +++ b/config/manager/params.env @@ -1,2 +1,2 @@ -codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.3.1 +codeflare-operator-controller-image=quay.io/opendatahub/codeflare-operator:v1.4.0 namespace=opendatahub diff --git a/go.mod b/go.mod index 57c47c46..f8a0da50 100644 --- a/go.mod +++ b/go.mod @@ -10,15 +10,15 @@ require ( github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/codeflare-common v0.0.0-20240207083912-d7a229270a0a - github.com/ray-project/kuberay/ray-operator v1.0.0 + github.com/ray-project/kuberay/ray-operator v1.1.0 go.uber.org/zap v1.26.0 - k8s.io/api v0.28.3 - k8s.io/apimachinery v0.28.3 + k8s.io/api v0.28.4 + k8s.io/apimachinery v0.28.4 k8s.io/client-go v11.0.0+incompatible - k8s.io/component-base v0.28.3 + k8s.io/component-base v0.28.4 k8s.io/klog/v2 v2.100.1 k8s.io/utils v0.0.0-20230726121419-3b25d923346b - sigs.k8s.io/controller-runtime v0.16.1 + sigs.k8s.io/controller-runtime v0.16.3 sigs.k8s.io/yaml v1.3.0 ) @@ -92,7 +92,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.28.2 // indirect + k8s.io/apiextensions-apiserver v0.28.4 // indirect k8s.io/kube-openapi v0.0.0-20230901164831-6c774f458599 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect diff --git a/go.sum b/go.sum index d67f55ff..10d47e58 100644 --- a/go.sum +++ b/go.sum @@ -211,8 +211,8 @@ github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqSc github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/ray-project/kuberay/ray-operator v1.0.0 h1:i69nvbV7az2FG41VHQgxrmhD+SUl8ca+ek4RPbSE2Q0= -github.com/ray-project/kuberay/ray-operator v1.0.0/go.mod h1:7C7ebIkxtkmOX8w1iiLrKM1j4hkZs/Guzm3WdePk/yg= +github.com/ray-project/kuberay/ray-operator v1.1.0 h1:kCbPZG6SDv11A1F1VR0oG1oCTjrKUuZiMDsyJqmNr3g= +github.com/ray-project/kuberay/ray-operator v1.1.0/go.mod h1:ZqyKKvMP5nKDldQoKmur+Wcx7wVlV9Q98phFqHzr+KY= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= @@ -402,18 +402,18 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= -k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= -k8s.io/apiextensions-apiserver v0.28.2 h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU= -k8s.io/apiextensions-apiserver v0.28.2/go.mod h1:5tnkxLGa9nefefYzWuAlWZ7RZYuN/765Au8cWLA6SRg= +k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY= +k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0= +k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU= +k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= -k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= +k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8= +k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= -k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= +k8s.io/component-base v0.28.4 h1:c/iQLWPdUgI90O+T9TeECg8o7N3YJTiuz2sKxILYcYo= +k8s.io/component-base v0.28.4/go.mod h1:m9hR0uvqXDybiGL2nf/3Lf0MerAfQXzkfWhUY58JUbU= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= @@ -431,8 +431,8 @@ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.16.1 h1:+15lzrmHsE0s2kNl0Dl8cTchI5Cs8qofo5PGcPrV9z0= -sigs.k8s.io/controller-runtime v0.16.1/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU= +sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= +sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= From 6be438df2ac73abe3d63f6522fdb294819b80cd3 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 30 Oct 2023 10:38:57 +0100 Subject: [PATCH 370/377] CARRY: Remove e2e and OLM upgrade tests --- .../resources-olm-upgrade/catalogsource.yaml | 12 -- .../resources-olm-upgrade/operatorgroup.yaml | 5 - .../resources-olm-upgrade/subscription.yaml | 19 -- .github/workflows/e2e_tests.yaml | 117 ------------ .github/workflows/olm_tests.yaml | 169 ------------------ 5 files changed, 322 deletions(-) delete mode 100644 .github/resources-olm-upgrade/catalogsource.yaml delete mode 100644 .github/resources-olm-upgrade/operatorgroup.yaml delete mode 100644 .github/resources-olm-upgrade/subscription.yaml delete mode 100644 .github/workflows/e2e_tests.yaml delete mode 100644 .github/workflows/olm_tests.yaml diff --git a/.github/resources-olm-upgrade/catalogsource.yaml b/.github/resources-olm-upgrade/catalogsource.yaml deleted file mode 100644 index c7449244..00000000 --- a/.github/resources-olm-upgrade/catalogsource.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: operators.coreos.com/v1alpha1 -kind: CatalogSource -metadata: - name: codeflare-olm-test - namespace: olm -spec: - displayName: '' - grpcPodConfig: - securityContextConfig: restricted - image: "${CATALOG_BASE_IMG}" - publisher: '' - sourceType: grpc diff --git a/.github/resources-olm-upgrade/operatorgroup.yaml b/.github/resources-olm-upgrade/operatorgroup.yaml deleted file mode 100644 index 1fa7fc7d..00000000 --- a/.github/resources-olm-upgrade/operatorgroup.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: openshift-operators - namespace: openshift-operators diff --git a/.github/resources-olm-upgrade/subscription.yaml b/.github/resources-olm-upgrade/subscription.yaml deleted file mode 100644 index 07fbb23b..00000000 --- a/.github/resources-olm-upgrade/subscription.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: codeflare-operator - namespace: openshift-operators -spec: - channel: alpha - installPlanApproval: Automatic - name: codeflare-operator - source: codeflare-olm-test - sourceNamespace: olm - config: - resources: - limits: - cpu: 400m - memory: 128Mi - requests: - cpu: 50m - memory: 64Mi diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml deleted file mode 100644 index 0a0ad397..00000000 --- a/.github/workflows/e2e_tests.yaml +++ /dev/null @@ -1,117 +0,0 @@ -name: e2e - -on: - pull_request: - branches: - - main - - 'release-*' - paths-ignore: - - 'docs/**' - - '**.adoc' - - '**.md' - - 'LICENSE' - push: - branches: - - main - - 'release-*' - paths-ignore: - - 'docs/**' - - '**.adoc' - - '**.md' - - 'LICENSE' - -concurrency: - group: ${{ github.head_ref }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - kubernetes-e2e: - - runs-on: ubuntu-20.04-4core - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Checkout common repo code - uses: actions/checkout@v4 - with: - repository: 'project-codeflare/codeflare-common' - ref: 'main' - path: 'common' - - - name: Set Go - uses: actions/setup-go@v5 - with: - go-version-file: './go.mod' - - - name: Set up gotestfmt - uses: gotesttools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup and start KinD cluster - uses: ./common/github-actions/kind - - - name: Deploy CodeFlare stack - id: deploy - run: | - echo Setting up CodeFlare stack - make setup-e2e - - echo Deploying CodeFlare operator - IMG="${REGISTRY_ADDRESS}"/codeflare-operator - make image-push -e IMG="${IMG}" - make deploy -e IMG="${IMG}" -e ENV="e2e" - kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager - - - name: Run e2e tests - run: | - export CODEFLARE_TEST_TIMEOUT_SHORT=1m - export CODEFLARE_TEST_TIMEOUT_MEDIUM=5m - export CODEFLARE_TEST_TIMEOUT_LONG=10m - export CODEFLARE_TEST_TIMEOUT_GPU_PROVISIONING=30m - - export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} - echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV - - set -euo pipefail - go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt - - - name: Print CodeFlare operator logs - if: always() && steps.deploy.outcome == 'success' - run: | - echo "Printing CodeFlare operator logs" - kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log - - - name: Print KubeRay operator logs - if: always() && steps.deploy.outcome == 'success' - run: | - echo "Printing KubeRay operator logs" - kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log - - - name: Export all KinD pod logs - uses: ./common/github-actions/kind-export-logs - if: always() && steps.deploy.outcome == 'success' - with: - output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} - - - name: Upload logs - uses: actions/upload-artifact@v4 - if: always() && steps.deploy.outcome == 'success' - with: - name: logs - retention-days: 10 - path: | - ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log - - - name: Post notification about failure to a Slack channel in case of push event - if: failure() && github.event_name == 'push' - uses: slackapi/slack-github-action@v1.24.0 - with: - channel-id: "codeflare-nightlies" - slack-message: "e2e test on push failed, " - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml deleted file mode 100644 index abce29d3..00000000 --- a/.github/workflows/olm_tests.yaml +++ /dev/null @@ -1,169 +0,0 @@ -# This workflow will build the CodeFlare Operator image and catalog containing bundle with this image, execute OLM upgrade tests using this catalog - -name: OLM Install and Upgrade - -on: - pull_request: - branches: - - main - - 'release-*' - paths-ignore: - - 'docs/**' - - '**.adoc' - - '**.md' - - 'LICENSE' - -concurrency: - group: ${{ github.head_ref }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - kubernetes-olm-upgrade: - runs-on: ubuntu-20.04-4core - timeout-minutes: 60 - env: - OLM_VERSION: v0.25.0 - VERSION: "v0.0.0-ghaction" # Need to supply some semver version for bundle to be properly generated - CATALOG_BASE_IMG: "registry.access.redhat.com/redhat/community-operator-index:v4.13" - CODEFLARE_TEST_TIMEOUT_SHORT: "1m" - CODEFLARE_TEST_TIMEOUT_MEDIUM: "5m" - CODEFLARE_TEST_TIMEOUT_LONG: "10m" - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # fetching also previous commits to get tags - - - name: Checkout common repo code - uses: actions/checkout@v4 - with: - repository: 'project-codeflare/codeflare-common' - ref: 'main' - path: 'common' - - - name: Set Go - uses: actions/setup-go@v5 - with: - go-version-file: './go.mod' - - - name: Set up gotestfmt - uses: gotesttools/gotestfmt-action@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup and start KinD cluster - uses: ./common/github-actions/kind - - - name: Install OLM - run: | - kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml - # wait for a while to be sure CRDs are installed - sleep 1 - kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/olm.yaml - echo Wait for default CatalogSource to start - kubectl wait -n ${{ env.CATALOG_SOURCE_NAMESPACE }} catalogsource/${{ env.CATALOG_SOURCE_NAME }} --for=jsonpath='{.status.connectionState.lastObservedState}'=READY --timeout=180s - env: - CATALOG_SOURCE_NAME: "operatorhubio-catalog" - CATALOG_SOURCE_NAMESPACE: "olm" - - - name: Create openshift-operator namespace and OperatorGroup - run: | - # Need to use openshift-operator namespace due to https://github.com/project-codeflare/codeflare-operator/issues/161 - kubectl create namespace openshift-operators - kubectl create -f .github/resources-olm-upgrade/operatorgroup.yaml - - - name: Deploy latest released CodeFlare operator from OLM - id: deploy - run: | - echo Deploying CodeFlare operator using Subscription - envsubst < .github/resources-olm-upgrade/catalogsource.yaml > ${{ env.TEMP_DIR }}/catalogsource.yaml - envsubst < .github/resources-olm-upgrade/subscription.yaml > ${{ env.TEMP_DIR }}/subscription.yaml - - kubectl create -f ${{ env.TEMP_DIR }}/catalogsource.yaml - - echo Wait for CatalogSource ${{ env.CATALOG_SOURCE_NAME }} to start - kubectl wait -n ${{ env.CATALOG_SOURCE_NAMESPACE }} catalogsource/${{ env.CATALOG_SOURCE_NAME }} --for=jsonpath='{.status.connectionState.lastObservedState}'=READY --timeout=180s - - kubectl create -f ${{ env.TEMP_DIR }}/subscription.yaml - - echo Waiting for Subscription to be ready - kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} subscription/${{ env.SUBSCRIPTION_NAME }} --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s - - echo Waiting for Deployment to be ready - timeout 60 bash -c 'until [[ $(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}') ]]; do sleep 5 && echo "$(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}')"; done' - kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} deployment/codeflare-operator-manager --for=condition=Available=true --timeout=60s - env: - CATALOG_SOURCE_NAME: "codeflare-olm-test" - CATALOG_SOURCE_NAMESPACE: "olm" - SUBSCRIPTION_NAME: "codeflare-operator" - SUBSCRIPTION_NAMESPACE: "openshift-operators" - - - name: Store latest CSV version as PREVIOUS_VERSION env variable (used for bundle build) - run: | - CSV_VERSION=$(kubectl get ClusterServiceVersion -l operators.coreos.com/codeflare-operator.openshift-operators='' -n openshift-operators -o json | jq -r .items[].spec.version) - echo "PREVIOUS_VERSION=v$CSV_VERSION" >> $GITHUB_ENV - - - name: Build operator and catalog image - run: | - make image-push - make bundle-build - make bundle-push - make catalog-build-from-index - make catalog-push - env: - IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator:v0.0.1" - BUNDLE_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-bundle:v0.0.1" - CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1" - OPM_BUNDLE_OPT: "--use-http" - BUNDLE_PUSH_OPT: "--tls-verify=false" - CATALOG_PUSH_OPT: "--tls-verify=false" - - - name: Update Operator to the built version - run: | - ORIGINAL_POD_NAME=$(kubectl get pod -l app.kubernetes.io/name=codeflare-operator -n openshift-operators -o json | jq -r .items[].metadata.name) - echo "Running old operator pod name is ${ORIGINAL_POD_NAME}" - - echo Updating custom CatalogSource image to the built CatalogSource with latest operator - kubectl patch CatalogSource codeflare-olm-test -n olm --type merge --patch "{\"spec\":{\"image\":\"${CATALOG_IMG}\"}}" - - echo Waiting for previous operator pod to get deleted - kubectl wait --timeout=120s --for=delete pod/${ORIGINAL_POD_NAME} -n openshift-operators - - echo Waiting for Subscription to be ready - kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} subscription/${{ env.SUBSCRIPTION_NAME }} --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s - - echo Waiting for Deployment to be ready - timeout 60 bash -c 'until [[ $(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}') ]]; do sleep 5 && echo "$(kubectl get deployment/codeflare-operator-manager -n '${{ env.SUBSCRIPTION_NAMESPACE }}')"; done' - kubectl wait -n ${{ env.SUBSCRIPTION_NAMESPACE }} deployment/codeflare-operator-manager --for=condition=Available=true --timeout=60s - - echo Checking that correct CSV is available - CSV_VERSION=$(kubectl get ClusterServiceVersion/codeflare-operator.${VERSION} -n openshift-operators -o json | jq -r .spec.version) - if [ "v${CSV_VERSION}" != "${VERSION}" ]; then - echo "CSV version v${CSV_VERSION} doesn't match expected version ${VERSION}" - exit 1 - fi - env: - CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1" - SUBSCRIPTION_NAME: "codeflare-operator" - SUBSCRIPTION_NAMESPACE: "openshift-operators" - - - name: Print CodeFlare operator logs - if: always() && steps.deploy.outcome == 'success' - run: | - echo "Printing CodeFlare operator logs" - mkdir logs - kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee logs/codeflare-operator.log - - - name: Export all KinD pod logs - uses: ./common/github-actions/kind-export-logs - if: always() && steps.deploy.outcome == 'success' - with: - output-directory: logs - - - name: Upload logs - uses: actions/upload-artifact@v4 - if: always() && steps.deploy.outcome == 'success' - with: - name: logs - retention-days: 10 - path: logs/**/*.log From 3433d042b281a777eb30592ac72a7714c9645ef0 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Thu, 12 Oct 2023 11:14:43 +0100 Subject: [PATCH 371/377] CARRY: Added automated workflow for pushing opendatahub/codeflare-operator image --- .github/workflows/build-and-push.yaml | 51 +++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/build-and-push.yaml diff --git a/.github/workflows/build-and-push.yaml b/.github/workflows/build-and-push.yaml new file mode 100644 index 00000000..919430c8 --- /dev/null +++ b/.github/workflows/build-and-push.yaml @@ -0,0 +1,51 @@ +# This workflow will build the CodeFlare Operator image and push it to the opendatahub image registry + +name: Build and Push + +on: + pull_request: + types: + - closed + workflow_dispatch: + +jobs: + build-and-push: + if: (github.event.pull_request.merged == true && github.event.pull_request.user.login == 'codeflare-machine-account' && contains(github.event.pull_request.title, 'Sync with Upstream')) || github.event_name == 'workflow_dispatch' + name: Build and push ODH/CFO image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set Go + uses: actions/setup-go@v3 + with: + go-version: v1.20 + + - name: Login to Quay.io + uses: redhat-actions/podman-login@v1 + with: + username: ${{ secrets.QUAY_BOT_USER }} + password: ${{ secrets.QUAY_BOT_PASSWORD }} + registry: quay.io + + - name: Get Upstream Release Tags + id: release-tags + run: | + release_tag=$(gh release view -R github.com/project-codeflare/codeflare-operator --json tagName | jq -r '.tagName') + echo "RELEASE_TAG=$release_tag" >> $GITHUB_ENV + env: + GITHUB_TOKEN: ${{ github.token }} + + - name: Cleanup the go.mod and go.sum + run: | + go mod tidy + + - name: Image Build and Push + run: | + make build + make image-build -e IMG=quay.io/opendatahub/codeflare-operator:${{ env.RELEASE_TAG }} + make image-push -e IMG=quay.io/opendatahub/codeflare-operator:${{ env.RELEASE_TAG }} + + - name: Delete remote branch + run: | + git push origin --delete sync-cfo-fork From 105366633e22630f2700b9ee3012ba431df9956d Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 17 Oct 2023 12:09:02 +0100 Subject: [PATCH 372/377] CARRY: Added Sync Fork workflow --- .github/workflows/sync-fork.yaml | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/sync-fork.yaml diff --git a/.github/workflows/sync-fork.yaml b/.github/workflows/sync-fork.yaml new file mode 100644 index 00000000..0c162a3a --- /dev/null +++ b/.github/workflows/sync-fork.yaml @@ -0,0 +1,39 @@ +# This workflow will create a sync pr for the opendatahub-io/codeflare-operator fork with project-codeflare/codeflare-operator + +name: Create Sync Fork PR + +on: + workflow_dispatch: + +jobs: + create-fork-sync-pr: + runs-on: ubuntu-latest + + env: + GIT_BRANCH: ${GITHUB_REF#refs/heads/} + + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false + - name: repo-sync + uses: repo-sync/github-sync@v2 + with: + source_repo: "https://github.com/project-codeflare/codeflare-operator.git" + source_branch: ${{ env.GIT_BRANCH }} + destination_branch: "sync-cfo-fork" + github_token: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + + - name: Set up Git + run: | + git config --global user.email "138894154+codeflare-machine-account@users.noreply.github.com" + git config --global user.name "codeflare-machine-account" + + - name: Create Pull Request + run: | + PR_TITLE="Sync with Upstream" + PR_BODY="This pull request updates the fork to match the latest changes from the upstream repository." + gh pr create --base "${{ env.GIT_BRANCH }}" --head sync-cfo-fork --title "$PR_TITLE" --body "$PR_BODY" + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash From df50ee736480b58ac175ae016008855b52f92bfc Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 22 Nov 2023 17:09:37 +0100 Subject: [PATCH 373/377] CARRY: Add Makefile and configuration files for e2e execution on OpenShift CI --- Makefile | 16 +++++++++++++ config/odh-operator/kustomization.yaml | 2 ++ config/odh-operator/odh.yaml | 13 +++++++++++ contrib/configuration/basic-dsc.yaml | 31 ++++++++++++++++++++++++++ 4 files changed, 62 insertions(+) create mode 100644 config/odh-operator/kustomization.yaml create mode 100644 config/odh-operator/odh.yaml create mode 100644 contrib/configuration/basic-dsc.yaml diff --git a/Makefile b/Makefile index 99e63324..61b524b7 100644 --- a/Makefile +++ b/Makefile @@ -194,6 +194,11 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi $(KUSTOMIZE) build config/${ENV} | kubectl delete --ignore-not-found=$(ignore-not-found) -f - git restore config/* +.PHONY: install-odh-operator +install-odh-operator: kustomize ## Install ODH operator into the OpenShift cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/odh-operator | kubectl apply -f - + kubectl wait -n openshift-operators subscription/opendatahub-operator --for=jsonpath='{.status.state}'=AtLatestKnown --timeout=180s + ##@ Build Dependencies ## Location to install dependencies to @@ -364,6 +369,17 @@ test-component: envtest ginkgo ## Run component tests. test-e2e: manifests fmt vet ## Run e2e tests. go test -timeout 30m -v ./test/e2e +.PHONY: test-odh +test-odh: manifests fmt vet ## Run e2e ODH tests. + go test -timeout 60m -v ./test/odh + +.PHONY: store-odh-logs +store-odh-logs: # Store all ODH relevant logs into artifact directory + kubectl logs -n opendatahub deployment/codeflare-operator-manager > ${ARTIFACT_DIR}/codeflare-operator.log + kubectl logs -n opendatahub deployment/kuberay-operator > ${ARTIFACT_DIR}/kuberay-operator.log + kubectl logs -n openshift-operators deployment/opendatahub-operator-controller-manager > ${ARTIFACT_DIR}/odh-operator.log + kubectl get events -n opendatahub > ${ARTIFACT_DIR}/odh-events.log + .PHONY: kind-e2e kind-e2e: ## Set up e2e KinD cluster test/e2e/kind.sh diff --git a/config/odh-operator/kustomization.yaml b/config/odh-operator/kustomization.yaml new file mode 100644 index 00000000..02358bd9 --- /dev/null +++ b/config/odh-operator/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- odh.yaml diff --git a/config/odh-operator/odh.yaml b/config/odh-operator/odh.yaml new file mode 100644 index 00000000..6f752159 --- /dev/null +++ b/config/odh-operator/odh.yaml @@ -0,0 +1,13 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: opendatahub-operator + labels: + operators.coreos.com/opendatahub-operator.openshift-operators: '' + namespace: openshift-operators +spec: + channel: fast + name: opendatahub-operator + installPlanApproval: Automatic + source: community-operators + sourceNamespace: openshift-marketplace diff --git a/contrib/configuration/basic-dsc.yaml b/contrib/configuration/basic-dsc.yaml new file mode 100644 index 00000000..1107cd31 --- /dev/null +++ b/contrib/configuration/basic-dsc.yaml @@ -0,0 +1,31 @@ +apiVersion: datasciencecluster.opendatahub.io/v1 +kind: DataScienceCluster +metadata: + labels: + app.kubernetes.io/created-by: opendatahub-operator + app.kubernetes.io/instance: default + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: datasciencecluster + app.kubernetes.io/part-of: opendatahub-operator + name: example-dsc +spec: + components: + codeflare: + devFlags: + manifests: + - uri: '' + contextDir: 'config' + sourcePath: 'manifests' + managementState: Managed + dashboard: + managementState: Managed + datasciencepipelines: + managementState: Removed + kserve: + managementState: Removed + modelmeshserving: + managementState: Removed + ray: + managementState: Managed + workbenches: + managementState: Managed From 865e713285c9b4a98cf1ff1a97897ced4f65c79b Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 28 Nov 2023 15:00:41 +0100 Subject: [PATCH 374/377] CARRY: Generate CodeFlare stack config map --- config/manager/kustomization.yaml | 7 +++++++ config/manager/manager.yaml | 5 ----- config/manager/params.yaml | 2 ++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 847eca17..bdb8b68d 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -12,6 +12,13 @@ configurations: - params.yaml vars: +- name: namespace + objref: + kind: ConfigMap + name: stack-config + apiVersion: v1 + fieldref: + fieldpath: data.namespace - name: codeflare_operator_controller_image objref: kind: ConfigMap diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 20a34fe9..14f62ad4 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -1,8 +1,3 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: system ---- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/config/manager/params.yaml b/config/manager/params.yaml index 43509ff2..4d4bd9d6 100644 --- a/config/manager/params.yaml +++ b/config/manager/params.yaml @@ -1,3 +1,5 @@ varReference: + - path: subjects[]/namespace + kind: ClusterRoleBinding - path: spec/template/spec/containers[]/image kind: Deployment From e4ed8f757e82ec49ae259058c022f6defa03e436 Mon Sep 17 00:00:00 2001 From: Wen Zhou Date: Wed, 25 Oct 2023 17:15:08 +0200 Subject: [PATCH 375/377] CARRY: update(manifests): use default namespace from ODH --- config/default/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 30b1273c..71d9fd58 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,5 +1,5 @@ # Adds namespace to all resources. -namespace: openshift-operators +namespace: opendatahub # Value of this field is prepended to the # names of all resources, e.g. a deployment named From c283d85c4eae2232a688bb4b68224b0da17d4df7 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Tue, 23 Jan 2024 10:34:22 +0100 Subject: [PATCH 376/377] PATCH: Adjust DSC source path for CodeFlare --- contrib/configuration/basic-dsc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/configuration/basic-dsc.yaml b/contrib/configuration/basic-dsc.yaml index 1107cd31..de992636 100644 --- a/contrib/configuration/basic-dsc.yaml +++ b/contrib/configuration/basic-dsc.yaml @@ -15,7 +15,7 @@ spec: manifests: - uri: '' contextDir: 'config' - sourcePath: 'manifests' + sourcePath: 'default' managementState: Managed dashboard: managementState: Managed From 46edabf618aa21e5636440ff4bd02ef6fc1421f9 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Mon, 4 Mar 2024 09:43:18 +0100 Subject: [PATCH 377/377] CARRY: Add workflow to release ODH/CFO with compiled test binaries --- .github/workflows/odh-release.yml | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/odh-release.yml diff --git a/.github/workflows/odh-release.yml b/.github/workflows/odh-release.yml new file mode 100644 index 00000000..900e910b --- /dev/null +++ b/.github/workflows/odh-release.yml @@ -0,0 +1,49 @@ +# This workflow will compile e2e tests and release them + +name: ODH Release +on: + workflow_dispatch: + inputs: + version: + description: 'Tag to be used for release, i.e.: v0.0.1' + required: true + +jobs: + release-odh: + runs-on: ubuntu-latest + + # Permission required to create a release + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + + - name: Set Go + uses: actions/setup-go@v5 + with: + go-version: v1.20 + + - name: Verify that release doesn't exist yet + shell: bash {0} + run: | + gh release view ${{ github.event.inputs.version }} + status=$? + if [[ $status -eq 0 ]]; then + echo "Release ${{ github.event.inputs.version }} already exists." + exit 1 + fi + env: + GITHUB_TOKEN: ${{ github.TOKEN }} + + - name: Compile tests + run: | + go test -c -o compiled-tests/e2e ./test/e2e/ + go test -c -o compiled-tests/odh ./test/odh/ + + - name: Creates a release in GitHub + run: | + gh release create ${{ github.event.inputs.version }} --target ${{ github.ref }} compiled-tests/* + env: + GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }} + shell: bash