From fb2102c5744ead9d9488261f96477208dd0edd4e Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 15 Jun 2023 14:52:53 +0200 Subject: [PATCH] OLM install and upgrade PR check for CodeFlare stack --- .github/actions/kind/action.yml | 49 +++++ .../e2e => .github/resources-kind}/kind.yaml | 0 .../resources-olm-upgrade/catalogsource.yaml | 12 ++ .../resources-olm-upgrade/operatorgroup.yaml | 5 + .../resources-olm-upgrade/subscription.yaml | 11 + .github/workflows/e2e_tests.yaml | 42 +--- .github/workflows/olm_tests.yaml | 190 ++++++++++++++++++ Makefile | 37 +++- 8 files changed, 303 insertions(+), 43 deletions(-) create mode 100644 .github/actions/kind/action.yml rename {test/e2e => .github/resources-kind}/kind.yaml (100%) create mode 100644 .github/resources-olm-upgrade/catalogsource.yaml create mode 100644 .github/resources-olm-upgrade/operatorgroup.yaml create mode 100644 .github/resources-olm-upgrade/subscription.yaml create mode 100644 .github/workflows/olm_tests.yaml diff --git a/.github/actions/kind/action.yml b/.github/actions/kind/action.yml new file mode 100644 index 000000000..58122e8a2 --- /dev/null +++ b/.github/actions/kind/action.yml @@ -0,0 +1,49 @@ +name: "Set up KinD" +description: "Step to start and configure KinD cluster" + +runs: + using: "composite" + steps: + - name: Init directories + shell: bash + run: | + TEMP_DIR="$(pwd)/tmp" + mkdir -p "${TEMP_DIR}" + echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV + + mkdir -p "$(pwd)/bin" + echo "$(pwd)/bin" >> $GITHUB_PATH + + - name: Container image registry + shell: bash + run: | + podman run -d -p 5000:5000 --name registry registry:2.8.1 + + export REGISTRY_ADDRESS=$(hostname -i):5000 + echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV + echo "Container image registry started at ${REGISTRY_ADDRESS}" + + KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml + echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV + envsubst < .github/resources-kind/kind.yaml > ${KIND_CONFIG_FILE} + + sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <> $GITHUB_ENV - - mkdir -p "$(pwd)/bin" - echo "$(pwd)/bin" >> $GITHUB_PATH - - name: Set Go uses: actions/setup-go@v3 with: @@ -73,37 +64,8 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - - name: Container image registry - run: | - podman run -d -p 5000:5000 --name registry registry:2.8.1 - - export REGISTRY_ADDRESS=$(hostname -i):5000 - echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV - echo "Container image registry started at ${REGISTRY_ADDRESS}" - - KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml - echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV - envsubst < ./test/e2e/kind.yaml > ${KIND_CONFIG_FILE} - - sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf < ${{ env.TEMP_DIR }}/catalogsource.yaml + envsubst < .github/resources-olm-upgrade/subscription.yaml > ${{ env.TEMP_DIR }}/subscription.yaml + + kubectl create -f ${{ env.TEMP_DIR }}/catalogsource.yaml + make wait-for-catalog-source + + kubectl create -f ${{ env.TEMP_DIR }}/subscription.yaml + + echo Waiting for Subscription to be ready + make wait-for-subscription + + echo Waiting for Deployment to be ready + make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators" + env: + CATALOG_SOURCE_NAME: "codeflare-olm-test" + CATALOG_SOURCE_NAMESPACE: "olm" + SUBSCRIPTION_NAME: "codeflare-operator" + SUBSCRIPTION_NAMESPACE: "openshift-operators" + + - name: Store latest CSV version as PREVIOUS_VERSION env variable (used for bundle build) + run: | + CSV_VERSION=$(kubectl get ClusterServiceVersion -l operators.coreos.com/codeflare-operator.openshift-operators='' -n openshift-operators -o json | jq -r .items[].spec.version) + echo "PREVIOUS_VERSION=v$CSV_VERSION" >> $GITHUB_ENV + + - name: Deploy CodeFlare stack (MCAD, KubeRay) + run: | + make setup-e2e + + - name: Build operator and catalog image + run: | + make image-push + make bundle-build + make bundle-push + make catalog-build-from-index + make catalog-push + env: + IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator:v0.0.1" + BUNDLE_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-bundle:v0.0.1" + CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1" + OPM_BUNDLE_OPT: "--use-http" + BUNDLE_PUSH_OPT: "--tls-verify=false" + CATALOG_PUSH_OPT: "--tls-verify=false" + + - name: Update Operator to the built version + run: | + ORIGINAL_POD_NAME=$(kubectl get pod -l app.kubernetes.io/name=codeflare-operator -n openshift-operators -o json | jq -r .items[].metadata.name) + echo "Running old operator pod name is ${ORIGINAL_POD_NAME}" + + echo Updating custom CatalogSource image to the built CatalogSource with latest operator + kubectl patch CatalogSource codeflare-olm-test -n olm --type merge --patch "{\"spec\":{\"image\":\"${CATALOG_IMG}\"}}" + + echo Waiting for previous operator pod to get deleted + kubectl wait --timeout=120s --for=delete pod/${ORIGINAL_POD_NAME} -n openshift-operators + + echo Waiting for Subscription to be ready + make wait-for-subscription + + echo Waiting for Deployment to be ready + make wait-for-deployment -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators" + + echo Checking that correct CSV is available + CSV_VERSION=$(kubectl get ClusterServiceVersion/codeflare-operator.${VERSION} -n openshift-operators -o json | jq -r .spec.version) + if [ "v${CSV_VERSION}" != "${VERSION}" ]; then + echo "CSV version v${CSV_VERSION} doesn't match expected version ${VERSION}" + exit 1 + fi + env: + CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1" + SUBSCRIPTION_NAME: "codeflare-operator" + SUBSCRIPTION_NAMESPACE: "openshift-operators" + + - name: Run e2e tests against built operator + run: | + export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} + echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV + + set -euo pipefail + go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt + + - name: Print CodeFlare operator logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing CodeFlare operator logs" + kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log + + - name: Print MCAD controller logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing MCAD controller logs" + kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log + + - name: Print KubeRay operator logs + if: always() && steps.deploy.outcome == 'success' + run: | + echo "Printing KubeRay operator logs" + kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log + + - name: Upload logs + uses: actions/upload-artifact@v3 + if: always() && steps.deploy.outcome == 'success' + with: + name: logs + retention-days: 10 + path: | + ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log diff --git a/Makefile b/Makefile index 5eb47e4c6..f10a87350 100644 --- a/Makefile +++ b/Makefile @@ -342,7 +342,7 @@ OPERATOR_SDK_DL_URL := https://github.com/operator-framework/operator-sdk/releas .PHONY: install-operator-sdk install-operator-sdk: $(OPERATOR_SDK) ## Download fixed version operator-sdk binary for consist outcome $(OPERATOR_SDK): $(LOCALBIN) - curl -L $(OPERATOR_SDK_DL_URL)/operator-sdk_$(shell go env GOOS)_$(shell go env GOARCH) --output-dir $(LOCALBIN) --output operator-sdk + curl -L $(OPERATOR_SDK_DL_URL)/operator-sdk_$(shell go env GOOS)_$(shell go env GOARCH) --output $(LOCALBIN)/operator-sdk chmod +x $(OPERATOR_SDK) .PHONY: validate-bundle @@ -366,7 +366,7 @@ bundle-build: bundle ## Build the bundle image. .PHONY: bundle-push bundle-push: ## Push the bundle image. - $(MAKE) image-push IMG=$(BUNDLE_IMG) + podman push $(BUNDLE_IMG) $(BUNDLE_PUSH_OPT) .PHONY: openshift-community-operator-release openshift-community-operator-release: install-gh-cli bundle ## build bundle and create PR in OpenShift community operators repository @@ -413,10 +413,21 @@ endif catalog-build: opm ## Build a catalog image. $(OPM) index add --container-tool podman --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) +# Build a catalog image by adding bundle images to existing catalog using the operator package manager tool, 'opm'. +.PHONY: catalog-build-from-index +catalog-build-from-index: opm ## Build a catalog image. + mkdir catalog + $(OPM) render $(CATALOG_BASE_IMG) -o yaml > catalog/bundles.yaml + $(OPM) render $(BUNDLE_IMG) $(OPM_BUNDLE_OPT) > catalog/codeflare-operator-bundle.yaml + sed -i -E "s/(.*)(- name: codeflare-operator.$(PREVIOUS_VERSION).*)/\1- name: codeflare-operator.$(VERSION)\n replaces: codeflare-operator.$(PREVIOUS_VERSION)\n\2/" catalog/bundles.yaml + $(OPM) validate catalog + $(OPM) generate dockerfile catalog + podman build . -f catalog.Dockerfile -t $(CATALOG_IMG) + # Push the catalog image. .PHONY: catalog-push catalog-push: ## Push a catalog image. - $(MAKE) image-push IMG=$(CATALOG_IMG) + podman push $(CATALOG_IMG) $(CATALOG_PUSH_OPT) .PHONY: test-unit test-unit: defaults manifests generate fmt vet envtest ## Run unit tests. @@ -429,3 +440,23 @@ test-e2e: defaults manifests generate fmt vet ## Run e2e tests. .PHONY: setup-e2e setup-e2e: ## Set up e2e tests. KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh + +# Wait until CatalogSource is in "READY" state +.PHONY: wait-for-catalog-source +wait-for-catalog-source: + timeout 120 bash -c 'while [[ "$$(kubectl get catalogsource/'$(CATALOG_SOURCE_NAME)' -n '$(CATALOG_SOURCE_NAMESPACE)' -o json | jq -r .status.connectionState.lastObservedState)" != "READY" ]]; do sleep 5 && echo "$$(kubectl get catalogsource/'$(CATALOG_SOURCE_NAME)' -n '$(CATALOG_SOURCE_NAMESPACE)' -o json | jq -r .status.connectionState.lastObservedState)" ; done' + +# Wait until Subscription is in "AtLatestKnown" state +.PHONY: wait-for-subscription +wait-for-subscription: + timeout 300 bash -c 'while [[ "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" != "AtLatestKnown" ]]; do sleep 5 && echo "$$(kubectl get subscription/'$(SUBSCRIPTION_NAME)' -n '$(SUBSCRIPTION_NAMESPACE)' -o json | jq -r .status.state)" ; done' + +# Default timeout for waiting actions +TIMEOUT ?= 180 + +# Wait until Deployment is in "Available" state +.PHONY: wait-for-deployment +wait-for-deployment: + # Wait until Deployment exists first, then use kubectl wait + timeout $(TIMEOUT) bash -c 'until [[ $$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)') ]]; do sleep 5 && echo "$$(kubectl get deployment/'$(DEPLOYMENT_NAME)' -n '$(DEPLOYMENT_NAMESPACE)')"; done' + kubectl wait --timeout=$(TIMEOUT)s --for=condition=Available=true deployment/$(DEPLOYMENT_NAME) -n $(DEPLOYMENT_NAMESPACE)