-
Notifications
You must be signed in to change notification settings - Fork 43
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
OLM install and upgrade PR check for CodeFlare stack
- Loading branch information
Showing
8 changed files
with
305 additions
and
43 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| name: "Set up KinD" | ||
| description: "Step to start and configure KinD cluster" | ||
|
|
||
| runs: | ||
| using: "composite" | ||
| steps: | ||
| - name: Init directories | ||
| shell: bash | ||
| run: | | ||
| TEMP_DIR="$(pwd)/tmp" | ||
| mkdir -p "${TEMP_DIR}" | ||
| echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV | ||
| mkdir -p "$(pwd)/bin" | ||
| echo "$(pwd)/bin" >> $GITHUB_PATH | ||
| - name: Container image registry | ||
| shell: bash | ||
| run: | | ||
| podman run -d -p 5000:5000 --name registry registry:2.8.1 | ||
| export REGISTRY_ADDRESS=$(hostname -i):5000 | ||
| echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV | ||
| echo "Container image registry started at ${REGISTRY_ADDRESS}" | ||
| KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml | ||
| echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV | ||
| envsubst < .github/resources-kind/kind.yaml > ${KIND_CONFIG_FILE} | ||
| sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <<EOF | ||
| [[registry]] | ||
| prefix = "$REGISTRY_ADDRESS" | ||
| insecure = true | ||
| location = "$REGISTRY_ADDRESS" | ||
| EOF' | ||
| - name: Setup KinD cluster | ||
| uses: helm/[email protected] | ||
| with: | ||
| cluster_name: cluster | ||
| version: v0.17.0 | ||
| config: ${{ env.KIND_CONFIG_FILE }} | ||
|
|
||
| - name: Print cluster info | ||
| shell: bash | ||
| run: | | ||
| echo "KinD cluster:" | ||
| kubectl cluster-info | ||
| kubectl describe nodes |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| apiVersion: operators.coreos.com/v1alpha1 | ||
| kind: CatalogSource | ||
| metadata: | ||
| name: codeflare-olm-test | ||
| namespace: olm | ||
| spec: | ||
| displayName: '' | ||
| grpcPodConfig: | ||
| securityContextConfig: restricted | ||
| image: "${CATALOG_BASE_IMG}" | ||
| publisher: '' | ||
| sourceType: grpc |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| apiVersion: operators.coreos.com/v1 | ||
| kind: OperatorGroup | ||
| metadata: | ||
| name: openshift-operators | ||
| namespace: openshift-operators |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| apiVersion: operators.coreos.com/v1alpha1 | ||
| kind: Subscription | ||
| metadata: | ||
| name: codeflare-operator | ||
| namespace: openshift-operators | ||
| spec: | ||
| channel: alpha | ||
| installPlanApproval: Automatic | ||
| name: codeflare-operator | ||
| source: codeflare-olm-test | ||
| sourceNamespace: olm | ||
| startingCSV: codeflare-operator.${PREVIOUS_VERSION} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -54,15 +54,6 @@ jobs: | |
| with: | ||
| submodules: recursive | ||
|
|
||
| - name: Init directories | ||
| run: | | ||
| TEMP_DIR="$(pwd)/tmp" | ||
| mkdir -p "${TEMP_DIR}" | ||
| echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV | ||
| mkdir -p "$(pwd)/bin" | ||
| echo "$(pwd)/bin" >> $GITHUB_PATH | ||
| - name: Set Go | ||
| uses: actions/setup-go@v3 | ||
| with: | ||
|
|
@@ -73,37 +64,8 @@ jobs: | |
| with: | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
|
|
||
| - name: Container image registry | ||
| run: | | ||
| podman run -d -p 5000:5000 --name registry registry:2.8.1 | ||
| export REGISTRY_ADDRESS=$(hostname -i):5000 | ||
| echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV | ||
| echo "Container image registry started at ${REGISTRY_ADDRESS}" | ||
| KIND_CONFIG_FILE=${{ env.TEMP_DIR }}/kind.yaml | ||
| echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV | ||
| envsubst < ./test/e2e/kind.yaml > ${KIND_CONFIG_FILE} | ||
| sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <<EOF | ||
| [[registry]] | ||
| prefix = "$REGISTRY_ADDRESS" | ||
| insecure = true | ||
| location = "$REGISTRY_ADDRESS" | ||
| EOF' | ||
| - name: Setup KinD cluster | ||
| uses: helm/[email protected] | ||
| with: | ||
| cluster_name: cluster | ||
| version: v0.17.0 | ||
| config: ${{ env.KIND_CONFIG_FILE }} | ||
|
|
||
| - name: Print cluster info | ||
| run: | | ||
| echo "KinD cluster:" | ||
| kubectl cluster-info | ||
| kubectl describe nodes | ||
| - name: Setup and start KinD cluster | ||
| uses: ./.github/actions/kind | ||
|
|
||
| - name: Deploy CodeFlare stack | ||
| id: deploy | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,196 @@ | ||
| # This workflow will build the CodeFlare Operator image and catalog containing bundle with this image, execute OLM upgrade tests using this catalog | ||
|
|
||
| name: OLM Install and Upgrade | ||
|
|
||
| on: | ||
| pull_request: | ||
| branches: | ||
| - main | ||
| - 'release-*' | ||
| paths-ignore: | ||
| - 'docs/**' | ||
| - '**.adoc' | ||
| - '**.md' | ||
| - 'LICENSE' | ||
|
|
||
| concurrency: | ||
| group: ${{ github.head_ref }}-${{ github.workflow }} | ||
| cancel-in-progress: true | ||
|
|
||
| jobs: | ||
| kubernetes: | ||
| runs-on: ubuntu-20.04 | ||
| timeout-minutes: 60 | ||
| env: | ||
| OLM_VERSION: v0.24.0 | ||
| VERSION: "v999.0.0" # Need to supply some semver version for bundle to be properly generated | ||
| CATALOG_BASE_IMG: "registry.access.redhat.com/redhat/community-operator-index:v4.13" | ||
| CODEFLARE_TEST_TIMEOUT_SHORT: "1m" | ||
| CODEFLARE_TEST_TIMEOUT_MEDIUM: "3m" | ||
| CODEFLARE_TEST_TIMEOUT_LONG: "10m" | ||
|
|
||
| steps: | ||
| - name: Cleanup | ||
| run: | | ||
| ls -lart | ||
| echo "Initial status:" | ||
| df -h | ||
| echo "Cleaning up resources:" | ||
| sudo swapoff -a | ||
| sudo rm -f /swapfile | ||
| sudo apt clean | ||
| sudo rm -rf /usr/share/dotnet | ||
| sudo rm -rf /opt/ghc | ||
| sudo rm -rf "/usr/local/share/boost" | ||
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | ||
| docker rmi $(docker image ls -aq) | ||
| echo "Final status:" | ||
| df -h | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| fetch-depth: 0 # fetching also previous commits to get tags | ||
|
|
||
| - name: Set Go | ||
| uses: actions/setup-go@v3 | ||
| with: | ||
| go-version: v1.18 | ||
|
|
||
| - name: Set up gotestfmt | ||
| uses: gotesttools/gotestfmt-action@v2 | ||
| with: | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
|
|
||
| - name: Store latest tag on branch | ||
| run: | | ||
| echo "PREVIOUS_VERSION=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV | ||
| - name: Setup and start KinD cluster | ||
| uses: ./.github/actions/kind | ||
|
|
||
| - name: Install OLM | ||
| run: | | ||
| kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/crds.yaml | ||
| # wait for a while to be sure CRDs are installed | ||
| sleep 1 | ||
| kubectl create -f https://github.com/operator-framework/operator-lifecycle-manager/releases/download/${OLM_VERSION}/olm.yaml | ||
| - name: Create openshift-operator namespace and OperatorGroup | ||
| run: | | ||
| # Need to use openshift-operator namespace due to https://github.com/project-codeflare/codeflare-operator/issues/161 | ||
| kubectl create namespace openshift-operators | ||
| kubectl create -f .github/resources-olm-upgrade/operatorgroup.yaml | ||
| - name: Deploy latest released CodeFlare operator from OLM | ||
| id: deploy | ||
| run: | | ||
| echo Deploying CodeFlare operator using Subscription | ||
| envsubst < .github/resources-olm-upgrade/catalogsource.yaml > ${{ env.TEMP_DIR }}/catalogsource.yaml | ||
| envsubst < .github/resources-olm-upgrade/subscription.yaml > ${{ env.TEMP_DIR }}/subscription.yaml | ||
| kubectl create -f ${{ env.TEMP_DIR }}/catalogsource.yaml | ||
| kubectl create -f ${{ env.TEMP_DIR }}/subscription.yaml | ||
| echo Waiting for Subscription to be ready | ||
| make subscription-ready | ||
| echo Waiting for Deployment to be ready | ||
| make deployment-available -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators" | ||
| echo Checking that correct CSV is available | ||
| CSV_VERSION=$(kubectl get ClusterServiceVersion/codeflare-operator.${PREVIOUS_VERSION} -n openshift-operators -o json | jq -r .spec.version) | ||
| if [ "v${CSV_VERSION}" != "${PREVIOUS_VERSION}" ]; then | ||
| echo "CSV version v${CSV_VERSION} doesn't match expected version ${PREVIOUS_VERSION}" | ||
| exit 1 | ||
| fi | ||
| env: | ||
| SUBSCRIPTION_NAME: "codeflare-operator" | ||
| SUBSCRIPTION_NAMESPACE: "openshift-operators" | ||
|
|
||
| - name: Deploy CodeFlare stack (MCAD, KubeRay) | ||
| run: | | ||
| make setup-e2e | ||
| - name: Run e2e tests | ||
| run: | | ||
| export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} | ||
| echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV | ||
| set -euo pipefail | ||
| go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest-original.log | gotestfmt | ||
| - name: Build operator and catalog image | ||
| run: | | ||
| make image-push | ||
| make bundle-build | ||
| make bundle-push | ||
| make catalog-build-from-index | ||
| make catalog-push | ||
| env: | ||
| IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator:v0.0.1" | ||
| BUNDLE_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-bundle:v0.0.1" | ||
| CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1" | ||
| OPM_BUNDLE_OPT: "--use-http" | ||
| BUNDLE_PUSH_OPT: "--tls-verify=false" | ||
| CATALOG_PUSH_OPT: "--tls-verify=false" | ||
|
|
||
| - name: Update Operator to the built version | ||
| run: | | ||
| ORIGINAL_POD_NAME=$(kubectl get pod -l app.kubernetes.io/name=codeflare-operator -n openshift-operators -o json | jq -r .items[].metadata.name) | ||
| echo "Running old operator pod name is ${ORIGINAL_POD_NAME}" | ||
| echo Updating custom CatalogSource image to the built CatalogSource with latest operator | ||
| kubectl patch CatalogSource codeflare-olm-test -n olm --type merge --patch "{\"spec\":{\"image\":\"${CATALOG_IMG}\"}}" | ||
| echo Waiting for previous operator pod to get deleted | ||
| kubectl wait --timeout=120s --for=delete pod/${ORIGINAL_POD_NAME} -n openshift-operators | ||
| echo Waiting for Subscription to be ready | ||
| make subscription-ready | ||
| echo Waiting for Deployment to be ready | ||
| make deployment-available -e TIMEOUT=60 -e DEPLOYMENT_NAME="codeflare-operator-manager" -e DEPLOYMENT_NAMESPACE="openshift-operators" | ||
| echo Checking that correct CSV is available | ||
| CSV_VERSION=$(kubectl get ClusterServiceVersion/codeflare-operator.${VERSION} -n openshift-operators -o json | jq -r .spec.version) | ||
| if [ "v${CSV_VERSION}" != "${VERSION}" ]; then | ||
| echo "CSV version v${CSV_VERSION} doesn't match expected version ${VERSION}" | ||
| exit 1 | ||
| fi | ||
| env: | ||
| CATALOG_IMG: "${{ env.REGISTRY_ADDRESS }}/codeflare-operator-catalog:v0.0.1" | ||
| SUBSCRIPTION_NAME: "codeflare-operator" | ||
| SUBSCRIPTION_NAMESPACE: "openshift-operators" | ||
|
|
||
| - name: Run e2e tests against built operator | ||
| run: | | ||
| set -euo pipefail | ||
| go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest-upgraded.log | gotestfmt | ||
| - name: Print CodeFlare operator logs | ||
| if: always() && steps.deploy.outcome == 'success' | ||
| run: | | ||
| echo "Printing CodeFlare operator logs" | ||
| kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log | ||
| - name: Print MCAD controller logs | ||
| if: always() && steps.deploy.outcome == 'success' | ||
| run: | | ||
| echo "Printing MCAD controller logs" | ||
| kubectl logs -n codeflare-system --tail -1 -l component=multi-cluster-application-dispatcher | tee ${CODEFLARE_TEST_OUTPUT_DIR}/mcad.log | ||
| - name: Print KubeRay operator logs | ||
| if: always() && steps.deploy.outcome == 'success' | ||
| run: | | ||
| echo "Printing KubeRay operator logs" | ||
| kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log | ||
| - name: Upload logs | ||
| uses: actions/upload-artifact@v3 | ||
| if: always() && steps.deploy.outcome == 'success' | ||
| with: | ||
| name: logs | ||
| retention-days: 10 | ||
| path: | | ||
| ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters