Update fms-hf-tuning e2e test

.github/workflows/odh-build-and-publish-operator-image.yaml

@@ -0,0 +1,125 @@
+# This is a copy of the publish-core-images.yaml and has been customized to
+# use the quay login credentials.
+# The unused parts of the original have been commented out on purpose.
+name: ODH
+
+on:
+  - push
+  - pull_request
+
+jobs:
+    build-and-publish-operator:
+      name: Build and (or) Publish Image
+      runs-on: ubuntu-latest  
+      env:
+        GOPATH: ${{ github.workspace }}/go
+        REPO_NAME: ${{ vars.QUAY_REPO_NAME || 'opendatahub' }}
+      steps:
+        - name: Environment dump
+          shell: bash
+          run: |
+            echo "GOPATH = ${GOPATH}"
+            echo "REPO_NAME = ${REPO_NAME}"
+
+        - name: Checkout
+          uses: actions/checkout@v4
+
+        - name: Set up Go
+          uses: actions/setup-go@v5
+          with:
+            go-version-file: go.mod
+
+        - name: Run go mod
+          shell: bash
+          run: |
+            go mod download
+
+        # Build operators inside the gh runner vm directly and then copy the go binaries to docker images using the Dockerfile.buildx
+        - name: Build linux/amd64 operator binary
+          env:
+            CGO_ENABLED: 1
+            GOOS: linux
+            GOARCH: amd64
+          shell: bash
+          run: |
+            go build -tags strictfipsruntime -a -o manager-$GOARCH cmd/training-operator.v1/main.go
+
+        - name: Build linux/arm64 operator binary
+          env:
+            CC: aarch64-linux-gnu-gcc
+            CGO_ENABLED: 1
+            GOOS: linux
+            GOARCH: arm64
+          shell: bash
+          run: |
+            sudo apt-get update
+            sudo apt-get install -y gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+            go build -tags strictfipsruntime -a -o manager-$GOARCH cmd/training-operator.v1/main.go
+
+        - name: Add docker tags
+          id: meta
+          uses: docker/metadata-action@v5
+          with:
+            images: quay.io/${{ env.REPO_NAME }}/training-operator
+            tags: |
+              type=raw,latest
+              type=ref,event=pr
+              type=sha,prefix=v1-odh-  
+
+        - name: Build image 
+          id: build-image
+          uses: redhat-actions/buildah-build@v2
+          with:
+            image: quay.io/${{ env.REPO_NAME }}/training-operator
+            tags: ${{ steps.meta.outputs.tags }}
+            labels: ${{ steps.meta.outputs.labels }}
+            platforms: linux/amd64,linux/arm64 
+            containerfiles: |
+              build/images/training-operator/Dockerfile.multiarch
+            extra-args: |
+              --pull  
+
+        # Check if image is build
+        - name: Check images created
+          shell: bash
+          run: buildah images | grep 'quay.io/${{ env.REPO_NAME }}/training-operator'
+
+        - name: Check image manifest
+          shell: bash
+          run: |
+            buildah manifest inspect ${{ steps.build-image.outputs.image }}:latest      
+
+
+        - name: Check image metadata
+          shell: bash  
+          run: |
+              buildah inspect ${{ steps.build-image.outputs.image-with-tag }} | jq '.OCIv1.config.Labels."org.opencontainers.image.title"'
+              buildah inspect ${{ steps.build-image.outputs.image-with-tag }} | jq '.OCIv1.config.Labels."org.opencontainers.image.description"'
+              buildah inspect ${{ steps.build-image.outputs.image-with-tag }} | jq '.Docker.config.Labels."org.opencontainers.image.title"'
+              buildah inspect ${{ steps.build-image.outputs.image-with-tag }} | jq '.Docker.config.Labels."org.opencontainers.image.description"'
+
+        - name: Login to Quay.io
+          id: podman-login-quay
+          # Trigger step only for specific branch (master, v.*-branch) or tag (v.*).
+          if:  (github.ref == 'refs/heads/dev' || (startsWith(github.ref, 'refs/heads/v') && endsWith(github.ref, '-branch')) || startsWith(github.ref, 'refs/tags/v')) 
+          shell: bash
+          run: |
+              podman login --username ${{ secrets.QUAY_USERNAME }} --password ${{ secrets.QUAY_TOKEN }} quay.io
+
+        - name: Push to Quay.io
+          if:  always() && steps.podman-login-quay.outcome == 'success' 
+          id: push-to-quay
+          uses: redhat-actions/push-to-registry@v2
+          with:
+            image: ${{ steps.build-image.outputs.image }}
+            tags: ${{ steps.build-image.outputs.tags }}
+
+        - name: Print image url
+          if: steps.push-to-quay.outcome == 'success'
+          shell: bash  
+          run: echo "Image pushed to ${{ steps.push-to-quay.outputs.registry-paths }}"
+
+        - name: Logout from Quay.io
+          if: always() && steps.podman-login-quay.outcome == 'success'
+          run: |
+            podman logout quay.io 

build/images/training-operator/Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.20 as builder
+FROM registry.access.redhat.com/ubi9/go-toolset:1.20.10 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -13,11 +13,13 @@ RUN go mod download
 COPY . .
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -a -o manager cmd/training-operator.v1/main.go
+USER root
+RUN CGO_ENABLED=1 GOOS=linux GO111MODULE=on go build -tags strictfipsruntime -a -o manager cmd/training-operator.v1/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details
-FROM gcr.io/distroless/static:latest
+FROM registry.access.redhat.com/ubi9/ubi-minimal:latest
 WORKDIR /
 COPY --from=builder /workspace/manager .
+USER 65532:65532
 ENTRYPOINT ["/manager"]

build/images/training-operator/Dockerfile.multiarch

@@ -0,0 +1,7 @@
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.9
+ARG TARGETARCH
+WORKDIR /
+COPY ./manager-${TARGETARCH} ./manager
+USER 65532:65532
+
+ENTRYPOINT ["/manager"]

go.mod

@@ -1,14 +1,15 @@
 module github.com/kubeflow/training-operator
 
-go 1.20
+go 1.22
 
 require (
 	github.com/go-logr/logr v1.4.1
 	github.com/google/go-cmp v0.6.0
-	github.com/onsi/ginkgo/v2 v2.14.0
-	github.com/onsi/gomega v1.30.0
+	github.com/onsi/ginkgo/v2 v2.15.0
+	github.com/onsi/gomega v1.31.1
+	github.com/project-codeflare/codeflare-common v0.0.0-20240404131119-10cc4982cd0e
 	github.com/prometheus/client_golang v1.18.0
-	github.com/sirupsen/logrus v1.9.0
+	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.8.4
 	go.uber.org/zap v1.26.0
 	k8s.io/api v0.29.3
@@ -19,63 +20,75 @@ require (
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
 	sigs.k8s.io/controller-runtime v0.17.2
+	sigs.k8s.io/kueue v0.6.1
 	sigs.k8s.io/scheduler-plugins v0.26.7
 	sigs.k8s.io/yaml v1.4.0
 	volcano.sh/apis v1.8.0
 )
 
 require (
+	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.8.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
-	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
+	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect
+	github.com/google/uuid v1.3.1 // indirect
+	github.com/gorilla/css v1.0.0 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/microcosm-cc/bluemonday v1.0.18 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/openshift-online/ocm-sdk-go v0.1.368 // indirect
+	github.com/openshift/api v0.0.0-20230213134911-7ba313770556 // indirect
+	github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/project-codeflare/appwrapper v0.7.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
+	github.com/ray-project/kuberay/ray-operator v1.1.0-alpha.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
 	golang.org/x/oauth2 v0.12.0 // indirect
 	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/term v0.16.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.16.1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.29.0 // indirect
-	k8s.io/component-base v0.29.0 // indirect
+	k8s.io/component-base v0.29.1 // indirect
 	k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect