OLM install and upgrade PR check for CodeFlare stack

project-codeflare · Jul 12, 2023 · 5811d7f · 5811d7f
1 parent 70f0719
commit 5811d7f
Show file tree

Hide file tree

Showing 8 changed files with 305 additions and 45 deletions.
diff --git a/.github/actions/kind/action.yml b/.github/actions/kind/action.yml
@@ -0,0 +1,49 @@
+name: "Set up KinD"
+description: "Step to start and configure KinD cluster"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Init directories
+      shell: bash
+      run: |
+        TEMP_DIR="$(pwd)/tmp"
+        mkdir -p "${TEMP_DIR}"
+        echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV
+
+        mkdir -p "$(pwd)/bin"
+        echo "$(pwd)/bin" >> $GITHUB_PATH
+
+    - name: Container image registry
+      shell: bash
+      run: |
+        podman run -d -p 5000:5000 --name registry registry:2.8.1
+
+        export REGISTRY_ADDRESS=$(hostname -i):5000
+        echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV
+        echo "Container image registry started at ${REGISTRY_ADDRESS}"
+
+        KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml
+        echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV
+        envsubst < .github/resources-kind/kind.yaml > ${KIND_CONFIG_FILE}
+
+        sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <<EOF
+        [[registry]]
+        prefix = "$REGISTRY_ADDRESS"
+        insecure = true
+        location = "$REGISTRY_ADDRESS"
+        EOF'
+
+    - name: Setup KinD cluster
+      uses: helm/[email protected]
+      with:
+        cluster_name: cluster
+        version: v0.17.0
+        config: ${{ env.KIND_CONFIG_FILE }}
+
+    - name: Print cluster info
+      shell: bash
+      run: |
+        echo "KinD cluster:"
+        kubectl cluster-info
+        kubectl describe nodes
diff --git a/test/e2e/kind.yaml → .github/resources-kind/kind.yaml b/test/e2e/kind.yaml → .github/resources-kind/kind.yaml
diff --git a/.github/resources-olm-upgrade/catalogsource.yaml b/.github/resources-olm-upgrade/catalogsource.yaml
@@ -0,0 +1,12 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: CatalogSource
+metadata:
+  name: codeflare-olm-test
+  namespace: olm
+spec:
+  displayName: ''
+  grpcPodConfig:
+    securityContextConfig: restricted
+  image: "${CATALOG_BASE_IMG}"
+  publisher: ''
+  sourceType: grpc
diff --git a/.github/resources-olm-upgrade/operatorgroup.yaml b/.github/resources-olm-upgrade/operatorgroup.yaml
@@ -0,0 +1,5 @@
+apiVersion: operators.coreos.com/v1
+kind: OperatorGroup
+metadata:
+  name: openshift-operators
+  namespace: openshift-operators
diff --git a/.github/resources-olm-upgrade/subscription.yaml b/.github/resources-olm-upgrade/subscription.yaml
@@ -0,0 +1,11 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: codeflare-operator
+  namespace: openshift-operators
+spec:
+  channel: alpha
+  installPlanApproval: Automatic
+  name: codeflare-operator
+  source: codeflare-olm-test
+  sourceNamespace: olm
diff --git a/.github/workflows/e2e_tests.yaml b/.github/workflows/e2e_tests.yaml
@@ -54,15 +54,6 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Init directories
-        run: |
-          TEMP_DIR="$(pwd)/tmp"
-          mkdir -p "${TEMP_DIR}"
-          echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV
-
-          mkdir -p "$(pwd)/bin"
-          echo "$(pwd)/bin" >> $GITHUB_PATH
-
       - name: Set Go
         uses: actions/setup-go@v3
         with:
@@ -73,37 +64,8 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Container image registry
-        run: |
-          podman run -d -p 5000:5000 --name registry registry:2.8.1
-
-          export REGISTRY_ADDRESS=$(hostname -i):5000
-          echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV
-          echo "Container image registry started at ${REGISTRY_ADDRESS}"
-
-          KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml
-          echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV
-          envsubst < ./test/e2e/kind.yaml > ${KIND_CONFIG_FILE}
-
-          sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <<EOF
-          [[registry]]
-          prefix = "$REGISTRY_ADDRESS"
-          insecure = true
-          location = "$REGISTRY_ADDRESS"
-          EOF'
-
-      - name: Setup KinD cluster
-        uses: helm/[email protected]
-        with:
-          cluster_name: cluster
-          version: v0.17.0
-          config: ${{ env.KIND_CONFIG_FILE }}
-
-      - name: Print cluster info
-        run: |
-          echo "KinD cluster:"
-          kubectl cluster-info
-          kubectl describe nodes
+      - name: Setup and start KinD cluster
+        uses: ./.github/actions/kind
 
       - name: Deploy CodeFlare stack
         id: deploy
@@ -120,8 +82,8 @@ jobs:
       - name: Run e2e tests
         run: |
           export CODEFLARE_TEST_TIMEOUT_SHORT=1m
-          export CODEFLARE_TEST_TIMEOUT_MEDIUM=3m
-          export CODEFLARE_TEST_TIMEOUT_LONG=8m
+          export CODEFLARE_TEST_TIMEOUT_MEDIUM=5m
+          export CODEFLARE_TEST_TIMEOUT_LONG=10m
 
           export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
           echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV

diff --git a/.github/workflows/olm_tests.yaml b/.github/workflows/olm_tests.yaml
@@ -0,0 +1,190 @@
+# This workflow will build the CodeFlare Operator image and catalog containing bundle with this image, execute OLM upgrade tests using this catalog
+
+name: OLM Install and Upgrade
+
+on:
+  pull_request:
+    branches:
+      - main
+      - 'release-*'
+    paths-ignore:
+      - 'docs/**'
+      - '**.adoc'
+      - '**.md'
+      - 'LICENSE'
+
+concurrency:
+  group: ${{ github.head_ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  kubernetes:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    env:
+      OLM_VERSION: v0.24.0
+      VERSION: "v0.0.0-ghaction"  # Need to supply some semver version for bundle to be properly generated
+      CATALOG_BASE_IMG: "registry.access.redhat.com/redhat/community-operator-index:v4.13"
+      CODEFLARE_TEST_TIMEOUT_SHORT: "1m"
+      CODEFLARE_TEST_TIMEOUT_MEDIUM: "5m"
+      CODEFLARE_TEST_TIMEOUT_LONG: "10m"
+
+    steps:
+      - name: Cleanup
+        run: |
+          ls -lart
+          echo "Initial status:"
+          df -h
+
+          echo "Cleaning up resources:"
+          sudo swapoff -a
+          sudo rm -f /swapfile
+          sudo apt clean
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker rmi $(docker image ls -aq)
+
+          echo "Final status:"
+          df -h
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # fetching also previous commits to get tags
+
+      - name: Set Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: v1.18
+
+      - name: Set up gotestfmt
+        uses: gotesttools/gotestfmt-action@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup and start KinD cluster
+        uses: ./.github/actions/kind
+
+      - name: Install OLM
+        run: |
+          kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml
+          # wait for a while to be sure CRDs are installed
+          sleep 1
+          kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/olm.yaml
+
+      - name: Create openshift-operator namespace and OperatorGroup
+        run: |
+          # Need to use openshift-operator namespace due to https://github.com/project-codeflare/codeflare-operator/issues/161
+          kubectl create namespace openshift-operators
+          kubectl create -f .github/resources-olm-upgrade/operatorgroup.yaml
+
+      - name: Deploy latest released CodeFlare operator from OLM
+        id: deploy
+        run: |
+          echo Deploying CodeFlare operator using Subscription
+          envsubst < .github/resources-olm-upgrade/catalogsource.yaml > ${{ env.TEMP_DIR }}/catalogsource.yaml
+          envsubst < .github/resources-olm-upgrade/subscription.yaml > ${{ env.TEMP_DIR }}/subscription.yaml
+
+          kubectl create -f ${{ env.TEMP_DIR }}/catalogsource.yaml
+          make wait-for-catalog-source
+
+          kubectl create -f ${{ env.TEMP_DIR }}/subscription.yaml
+
+          echo Waiting for Subscription to be ready
+          make wait-for-subscription
+
+          echo Waiting for Deployment to be ready
+          make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators"
+        env:
+          CATALOG_SOURCE_NAME: "codeflare-olm-test"
+          CATALOG_SOURCE_NAMESPACE: "olm"
+          SUBSCRIPTION_NAME: "codeflare-operator"
+          SUBSCRIPTION_NAMESPACE: "openshift-operators"
+
+      - name: Store latest CSV version as PREVIOUS_VERSION env variable (used for bundle build)
+        run: |
+          CSV_VERSION=$(kubectl get ClusterServiceVersion -l operators.coreos.com/codeflare-operator.openshift-operators='' -n openshift-operators -o json | jq -r .items[].spec.version)
+          echo "PREVIOUS_VERSION=v$CSV_VERSION" >> $GITHUB_ENV
+
+      - name: Deploy CodeFlare stack (MCAD, KubeRay)
+        run: |
+          make setup-e2e
+
+      - name: Build operator and catalog image
+        run: |
+          make image-push
+          make bundle-build
+          make bundle-push
+          make catalog-build-from-index
+          make catalog-push
+        env:
+          IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator:v0.0.1"
+          BUNDLE_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-bundle:v0.0.1"
+          CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1"
+          OPM_BUNDLE_OPT: "--use-http"
+          BUNDLE_PUSH_OPT: "--tls-verify=false"
+          CATALOG_PUSH_OPT: "--tls-verify=false"
+
+      - name: Update Operator to the built version
+        run: |
+          ORIGINAL_POD_NAME=$(kubectl get pod -l app.kubernetes.io/name=codeflare-operator -n openshift-operators -o json | jq -r .items[].metadata.name)
+          echo "Running old operator pod name is ${ORIGINAL_POD_NAME}"
+
+          echo Updating custom CatalogSource image to the built CatalogSource with latest operator
+          kubectl patch CatalogSource codeflare-olm-test -n olm --type merge --patch "{\"spec\":{\"image\":\"${CATALOG_IMG}\"}}"
+
+          echo Waiting for previous operator pod to get deleted
+          kubectl wait --timeout=120s --for=delete pod/${ORIGINAL_POD_NAME} -n openshift-operators
+
+          echo Waiting for Subscription to be ready
+          make wait-for-subscription
+
+          echo Waiting for Deployment to be ready
+          make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators"
+
+          echo Checking that correct CSV is available
+          CSV_VERSION=$(kubectl get ClusterServiceVersion/codeflare-operator.${VERSION} -n openshift-operators -o json | jq -r .spec.version)
+          if [ "v${CSV_VERSION}" != "${VERSION}" ]; then
+            echo "CSV version v${CSV_VERSION} doesn't match expected version ${VERSION}"
+            exit 1
+          fi
+        env:
+          CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1"
+          SUBSCRIPTION_NAME: "codeflare-operator"
+          SUBSCRIPTION_NAMESPACE: "openshift-operators"
+
+      - name: Run e2e tests against built operator
+        run: |
+          export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
+          echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
+
+          set -euo pipefail
+          go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
+
+      - name: Print CodeFlare operator logs
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing CodeFlare operator logs"
+          kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log
+
+      - name: Print MCAD controller logs
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing MCAD controller logs"
+          kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log
+
+      - name: Print KubeRay operator logs
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing KubeRay operator logs"
+          kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
+
+      - name: Upload logs
+        uses: actions/upload-artifact@v3
+        if: always() && steps.deploy.outcome == 'success'
+        with:
+          name: logs
+          retention-days: 10
+          path: |
+            ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log