From f2952d0a4a85f245753d3a5427edbbf8ebc5e7f9 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 13 May 2021 10:15:57 +0300 Subject: [PATCH 01/51] add community checks to operator --- .github/workflow/csi_operator_ci.yaml | 226 ++++++++++++++++++++++++++ build/ci/op-test.sh | 9 + build/ci/prepare_env.sh | 20 +++ build/ci/update_csv.sh | 11 ++ 4 files changed, 266 insertions(+) create mode 100644 .github/workflow/csi_operator_ci.yaml create mode 100755 build/ci/op-test.sh create mode 100755 build/ci/prepare_env.sh create mode 100755 build/ci/update_csv.sh diff --git a/.github/workflow/csi_operator_ci.yaml b/.github/workflow/csi_operator_ci.yaml new file mode 100644 index 000000000..0d9b55bcd --- /dev/null +++ b/.github/workflow/csi_operator_ci.yaml @@ -0,0 +1,226 @@ +name: Operator test + +env: + OP_TEST_DEBUG: 1 + OP_TEST_CONTAINER_OPT: "-t" +on: + push: + pull_request: + - develop +jobs: + operator_image_build: + name: "set env" + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + outputs: + operator_image: "csiblock1/ibm-block-csi-operator-amd64:new" + + prepare_env: + name: "prepare_env" + needs: operator_image_build + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: python -m pip install --upgrade pip yq + - uses: FranzDiebold/github-env-vars-action@v2 + - name: prepare_env + id: environment_setup + env: + operator_image: ${{ needs.operator_image_build.outputs.operator_image }} + run: | + build/ci/prepare_env.sh + outputs: + current_operator_image: "${{ steps.environment_setup.outputs.current_operator_image }}" + wanted_operator_image: "${{ steps.environment_setup.outputs.wanted_operator_image }}" + csv_file: "${{ steps.environment_setup.outputs.csv_file }}" + community_operators_path: "${{ steps.environment_setup.outputs.community_operators_path }}" + upstream_community_operators_path: "${{ steps.environment_setup.outputs.upstream_community_operators_path }}" + repository_path: "${{ steps.environment_setup.outputs.repository_path }}" + + CI-csi-kiwi: + name: "kiwi / Full operator test" + needs: prepare_env + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: update_csv + run: | + build/ci/update_csv.sh + env: + current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} + wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} + csv_file: ${{ needs.prepare_env.outputs.csv_file }} + repository_path: ${{ needs.prepare_env.outputs.repository_path }} + - name: Operator test + run: | + echo "kiwi ${{ needs.prepare_env.outputs.upstream_community_operators_path }}" + build/ci/op-test.sh kiwi "${{ needs.prepare_env.outputs.upstream_community_operators_path }}" + + CI-csi-lemon: + name: "lemon / Deploy from scratch" + needs: prepare_env + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: update_csv + run: | + build/ci/update_csv.sh + env: + current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} + wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} + csv_file: ${{ needs.prepare_env.outputs.csv_file }} + repository_path: ${{ needs.prepare_env.outputs.repository_path }} + - name: Operator test + run: | + echo "lemon ${{ needs.prepare_env.outputs.upstream_community_operators_path }}" + build/ci/op-test.sh lemon "${{ needs.prepare_env.outputs.upstream_community_operators_path }}" + + CI-csi-lemon-openshift_4_6: + name: "lemon / Deploy from scratch (v4.6)" + needs: prepare_env + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: update_csv + run: | + build/ci/update_csv.sh + env: + current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} + wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} + csv_file: ${{ needs.prepare_env.outputs.csv_file }} + repository_path: ${{ needs.prepare_env.outputs.repository_path }} + - name: Operator test + run: | + echo "lemon_v4.6 ${{ needs.prepare_env.outputs.community_operators_path }}" + build/ci/op-test.sh lemon_v4.6 "${{ needs.prepare_env.outputs.community_operators_path }}" + + CI-csi-lemon-openshift_4_7: + name: "lemon / Deploy from scratch (v4.7)" + needs: prepare_env + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: update_csv + run: | + build/ci/update_csv.sh + env: + current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} + wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} + csv_file: ${{ needs.prepare_env.outputs.csv_file }} + repository_path: ${{ needs.prepare_env.outputs.repository_path }} + - name: Operator test + run: | + echo "lemon_v4.7 ${{ needs.prepare_env.outputs.community_operators_path }}" + build/ci/op-test.sh lemon_v4.7 "${{ needs.prepare_env.outputs.community_operators_path }}" + + CI-csi-lemon-openshift_4_8: + name: "lemon / Deploy from scratch (v4.8)" + needs: prepare_env + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: update_csv + run: | + build/ci/update_csv.sh + env: + current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} + wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} + csv_file: ${{ needs.prepare_env.outputs.csv_file }} + repository_path: ${{ needs.prepare_env.outputs.repository_path }} + - name: Operator test + run: | + echo "lemon_v4.8 ${{ needs.prepare_env.outputs.community_operators_path }}" + build/ci/op-test.sh lemon_v4.8 "${{ needs.prepare_env.outputs.community_operators_path }}" + + CI-csi-orange-k8s: + name: "orange / Deploy k8s latest" + needs: prepare_env + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: update_csv + run: | + build/ci/update_csv.sh + env: + current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} + wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} + csv_file: ${{ needs.prepare_env.outputs.csv_file }} + repository_path: ${{ needs.prepare_env.outputs.repository_path }} + - name: Operator test + run: | + echo "orange ${{ needs.prepare_env.outputs.upstream_community_operators_path }}" + build/ci/op-test.sh orange "${{ needs.prepare_env.outputs.upstream_community_operators_path }}" + + CI-csi-orange-openshift_4_6: + name: "orange / Deploy openshift v4.6" + needs: prepare_env + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: update_csv + run: | + build/ci/update_csv.sh + env: + current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} + wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} + csv_file: ${{ needs.prepare_env.outputs.csv_file }} + repository_path: ${{ needs.prepare_env.outputs.repository_path }} + - name: Operator test + run: | + echo "orange_v4.6 ${{ needs.prepare_env.outputs.community_operators_path }}" + build/ci/op-test.sh orange_v4.6 "${{ needs.prepare_env.outputs.community_operators_path }}" + + CI-csi-orange-openshift_4_7: + name: "orange / Deploy openshift v4.7" + needs: prepare_env + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: update_csv + run: | + build/ci/update_csv.sh + env: + current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} + wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} + csv_file: ${{ needs.prepare_env.outputs.csv_file }} + repository_path: ${{ needs.prepare_env.outputs.repository_path }} + - name: Operator test + run: | + echo "orange_v4.7 ${{ needs.prepare_env.outputs.community_operators_path }}" + build/ci/op-test.sh orange_v4.7 "${{ needs.prepare_env.outputs.community_operators_path }}" + + CI-csi-orange-openshift_4_8: + name: "orange / Deploy openshift v4.8" + needs: prepare_env + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: update_csv + run: | + build/ci/update_csv.sh + env: + current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} + wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} + csv_file: ${{ needs.prepare_env.outputs.csv_file }} + repository_path: ${{ needs.prepare_env.outputs.repository_path }} + - name: Operator test + run: | + echo "orange_v4.8 ${{ needs.prepare_env.outputs.community_operators_path }}" + build/ci/op-test.sh orange_v4.8 "${{ needs.prepare_env.outputs.community_operators_path }}" \ No newline at end of file diff --git a/build/ci/op-test.sh b/build/ci/op-test.sh new file mode 100755 index 000000000..c4e9ffad9 --- /dev/null +++ b/build/ci/op-test.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +OP_TEST_SCRIPT_URL=${OP_TEST_SCRIPT_URL-"https://raw.githubusercontent.com/operator-framework/operator-test-playbooks/master/upstream/test/test.sh"} + +bash <(curl -sL $OP_TEST_SCRIPT_URL) $* +rc=$? +echo "rc=$rc" +[[ $rc -eq 0 ]] || { echo "Error: rc=$rc"; exit $rc; } diff --git a/build/ci/prepare_env.sh b/build/ci/prepare_env.sh new file mode 100755 index 000000000..56651cf67 --- /dev/null +++ b/build/ci/prepare_env.sh @@ -0,0 +1,20 @@ +#!/bin/bash -xe +set +o pipefail + +export csv_version=`yq .channels[0].currentCSV deploy/olm-catalog/ibm-block-csi-operator-community/ibm-block-csi-operator.package.yaml` +export csv_version=`echo ${csv_version//ibm-block-csi-operator.v}` +export csv_version=`echo ${csv_version//\"}` +export repository_path=~/work/$CI_REPOSITORY_NAME/$CI_REPOSITORY_NAME +export csv_file=$repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$csv_version/ibm-block-csi-operator.v$csv_version.clusterserviceversion.yaml +export wanted_image=$operator_image +export current_image=`yq .metadata.annotations.containerImage $csv_file` +export current_image=`echo ${current_image//\"}` +export upstream_community_operators_path=upstream-community-operators/ibm-block-csi-operator-community/$csv_version +export community_operators_path=community-operators/ibm-block-csi-operator-community/$csv_version + +echo "::set-output name=current_operator_image::${current_image}" +echo "::set-output name=wanted_operator_image::${wanted_image}" +echo "::set-output name=csv_file::${csv_file}" +echo "::set-output name=upstream_community_operators_path::${upstream_community_operators_path}" +echo "::set-output name=community_operators_path::${community_operators_path}" +echo "::set-output name=repository_path::${repository_path}" diff --git a/build/ci/update_csv.sh b/build/ci/update_csv.sh new file mode 100755 index 000000000..8c720508a --- /dev/null +++ b/build/ci/update_csv.sh @@ -0,0 +1,11 @@ +#!/bin/bash -xe +set +o pipefail + +sed -i "s+$current_operator_image+$wanted_operator_image+g" $csv_file +sed -i 's/.*minKubeVersion.*/ minKubeVersion: 1.19.0/' $csv_file + +mkdir $repository_path/upstream-community-operators/ +mkdir $repository_path/community-operators/ + +cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/ $repository_path/upstream-community-operators/ +cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/ $repository_path/community-operators/ From 9650ea5bb18d61b359535f0cad10e0d3d78fcda8 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Sun, 16 May 2021 10:13:11 +0300 Subject: [PATCH 02/51] add code review answers --- .github/workflow/csi_operator_ci.yaml | 4 +++- build/ci/op-test.sh | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflow/csi_operator_ci.yaml b/.github/workflow/csi_operator_ci.yaml index 0d9b55bcd..be87a7cd9 100644 --- a/.github/workflow/csi_operator_ci.yaml +++ b/.github/workflow/csi_operator_ci.yaml @@ -7,6 +7,7 @@ on: push: pull_request: - develop + - master jobs: operator_image_build: name: "set env" @@ -52,6 +53,7 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + # reduce code repetition in CSI-2992 ticket - name: update_csv run: | build/ci/update_csv.sh @@ -223,4 +225,4 @@ jobs: - name: Operator test run: | echo "orange_v4.8 ${{ needs.prepare_env.outputs.community_operators_path }}" - build/ci/op-test.sh orange_v4.8 "${{ needs.prepare_env.outputs.community_operators_path }}" \ No newline at end of file + build/ci/op-test.sh orange_v4.8 "${{ needs.prepare_env.outputs.community_operators_path }}" diff --git a/build/ci/op-test.sh b/build/ci/op-test.sh index c4e9ffad9..d5bc5800c 100755 --- a/build/ci/op-test.sh +++ b/build/ci/op-test.sh @@ -5,5 +5,4 @@ OP_TEST_SCRIPT_URL=${OP_TEST_SCRIPT_URL-"https://raw.githubusercontent.com/opera bash <(curl -sL $OP_TEST_SCRIPT_URL) $* rc=$? -echo "rc=$rc" -[[ $rc -eq 0 ]] || { echo "Error: rc=$rc"; exit $rc; } +[[ $rc -eq 0 ]] && echo "rc=$rc" || { echo "Error: rc=$rc"; exit $rc; } From 58a46c11d896fbaa20a85d4d4d9dcf4f7650923a Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Tue, 18 May 2021 18:21:28 +0300 Subject: [PATCH 03/51] complete the CI --- .../csi_operator_ci.yaml | 0 categories.json | 20 +++++++++++++++++++ .../ibm-block-csi-operator-community/ci.yaml | 3 +++ 3 files changed, 23 insertions(+) rename .github/{workflow => workflows}/csi_operator_ci.yaml (100%) create mode 100644 categories.json create mode 100644 deploy/olm-catalog/ibm-block-csi-operator-community/ci.yaml diff --git a/.github/workflow/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml similarity index 100% rename from .github/workflow/csi_operator_ci.yaml rename to .github/workflows/csi_operator_ci.yaml diff --git a/categories.json b/categories.json new file mode 100644 index 000000000..a0b2f49a7 --- /dev/null +++ b/categories.json @@ -0,0 +1,20 @@ +{ + "categories":[ + "AI/Machine Learning", + "Application Runtime", + "Big Data", + "Cloud Provider", + "Developer Tools", + "Database", + "Drivers and plugins", + "Integration & Delivery", + "Logging & Tracing", + "Modernization & Migration", + "Monitoring", + "Networking", + "OpenShift Optional", + "Security", + "Storage", + "Streaming & Messaging" + ] + } \ No newline at end of file diff --git a/deploy/olm-catalog/ibm-block-csi-operator-community/ci.yaml b/deploy/olm-catalog/ibm-block-csi-operator-community/ci.yaml new file mode 100644 index 000000000..337cb631a --- /dev/null +++ b/deploy/olm-catalog/ibm-block-csi-operator-community/ci.yaml @@ -0,0 +1,3 @@ +--- +# Use `replaces-mode` or `semver-mode`. Once you switch to `semver-mode`, there is no easy way back. +updateGraph: replaces-mode \ No newline at end of file From 135b96f8ba7b8b2e644b6a783d159c6a8f9f2444 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Tue, 18 May 2021 18:34:49 +0300 Subject: [PATCH 04/51] remove triger on push --- .github/workflows/csi_operator_ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index be87a7cd9..91bfcfa1b 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -4,10 +4,10 @@ env: OP_TEST_DEBUG: 1 OP_TEST_CONTAINER_OPT: "-t" on: - push: - pull_request: - - develop - - master + pull_request: + branches: + - develop + - master jobs: operator_image_build: name: "set env" From 352247c243f8dd91047b0ba6a123387acd678902 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 9 Jun 2021 12:47:38 +0300 Subject: [PATCH 05/51] use the fork way --- .github/workflows/csi_operator_ci.yaml | 193 ++---------------- build/ci/clean_demo_pr.sh | 42 ++++ build/ci/create_demo_pr.sh | 44 ++++ build/ci/op-test.sh | 8 - build/ci/update_csv.sh | 11 - categories.json | 20 -- .../ibm-block-csi-operator-community/ci.yaml | 3 - 7 files changed, 104 insertions(+), 217 deletions(-) create mode 100755 build/ci/clean_demo_pr.sh create mode 100755 build/ci/create_demo_pr.sh delete mode 100755 build/ci/op-test.sh delete mode 100755 build/ci/update_csv.sh delete mode 100644 categories.json delete mode 100644 deploy/olm-catalog/ibm-block-csi-operator-community/ci.yaml diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 91bfcfa1b..8d9d001f0 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -1,13 +1,8 @@ -name: Operator test - -env: - OP_TEST_DEBUG: 1 - OP_TEST_CONTAINER_OPT: "-t" +name: Community Operators test on: - pull_request: + push: branches: - - develop - - master + - task/CSI-2892_cert_community_operator_hub jobs: operator_image_build: name: "set env" @@ -46,183 +41,31 @@ jobs: upstream_community_operators_path: "${{ steps.environment_setup.outputs.upstream_community_operators_path }}" repository_path: "${{ steps.environment_setup.outputs.repository_path }}" - CI-csi-kiwi: - name: "kiwi / Full operator test" - needs: prepare_env - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - # reduce code repetition in CSI-2992 ticket - - name: update_csv - run: | - build/ci/update_csv.sh - env: - current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} - wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} - csv_file: ${{ needs.prepare_env.outputs.csv_file }} - repository_path: ${{ needs.prepare_env.outputs.repository_path }} - - name: Operator test - run: | - echo "kiwi ${{ needs.prepare_env.outputs.upstream_community_operators_path }}" - build/ci/op-test.sh kiwi "${{ needs.prepare_env.outputs.upstream_community_operators_path }}" - - CI-csi-lemon: - name: "lemon / Deploy from scratch" - needs: prepare_env - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: update_csv - run: | - build/ci/update_csv.sh - env: - current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} - wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} - csv_file: ${{ needs.prepare_env.outputs.csv_file }} - repository_path: ${{ needs.prepare_env.outputs.repository_path }} - - name: Operator test - run: | - echo "lemon ${{ needs.prepare_env.outputs.upstream_community_operators_path }}" - build/ci/op-test.sh lemon "${{ needs.prepare_env.outputs.upstream_community_operators_path }}" - - CI-csi-lemon-openshift_4_6: - name: "lemon / Deploy from scratch (v4.6)" - needs: prepare_env - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: update_csv - run: | - build/ci/update_csv.sh - env: - current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} - wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} - csv_file: ${{ needs.prepare_env.outputs.csv_file }} - repository_path: ${{ needs.prepare_env.outputs.repository_path }} - - name: Operator test - run: | - echo "lemon_v4.6 ${{ needs.prepare_env.outputs.community_operators_path }}" - build/ci/op-test.sh lemon_v4.6 "${{ needs.prepare_env.outputs.community_operators_path }}" - - CI-csi-lemon-openshift_4_7: - name: "lemon / Deploy from scratch (v4.7)" - needs: prepare_env - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: update_csv - run: | - build/ci/update_csv.sh - env: - current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} - wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} - csv_file: ${{ needs.prepare_env.outputs.csv_file }} - repository_path: ${{ needs.prepare_env.outputs.repository_path }} - - name: Operator test - run: | - echo "lemon_v4.7 ${{ needs.prepare_env.outputs.community_operators_path }}" - build/ci/op-test.sh lemon_v4.7 "${{ needs.prepare_env.outputs.community_operators_path }}" - - CI-csi-lemon-openshift_4_8: - name: "lemon / Deploy from scratch (v4.8)" - needs: prepare_env - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: update_csv - run: | - build/ci/update_csv.sh - env: - current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} - wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} - csv_file: ${{ needs.prepare_env.outputs.csv_file }} - repository_path: ${{ needs.prepare_env.outputs.repository_path }} - - name: Operator test - run: | - echo "lemon_v4.8 ${{ needs.prepare_env.outputs.community_operators_path }}" - build/ci/op-test.sh lemon_v4.8 "${{ needs.prepare_env.outputs.community_operators_path }}" - - CI-csi-orange-k8s: - name: "orange / Deploy k8s latest" - needs: prepare_env - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: update_csv - run: | - build/ci/update_csv.sh - env: - current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} - wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} - csv_file: ${{ needs.prepare_env.outputs.csv_file }} - repository_path: ${{ needs.prepare_env.outputs.repository_path }} - - name: Operator test - run: | - echo "orange ${{ needs.prepare_env.outputs.upstream_community_operators_path }}" - build/ci/op-test.sh orange "${{ needs.prepare_env.outputs.upstream_community_operators_path }}" - - CI-csi-orange-openshift_4_6: - name: "orange / Deploy openshift v4.6" - needs: prepare_env - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: update_csv - run: | - build/ci/update_csv.sh - env: - current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} - wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} - csv_file: ${{ needs.prepare_env.outputs.csv_file }} - repository_path: ${{ needs.prepare_env.outputs.repository_path }} - - name: Operator test - run: | - echo "orange_v4.6 ${{ needs.prepare_env.outputs.community_operators_path }}" - build/ci/op-test.sh orange_v4.6 "${{ needs.prepare_env.outputs.community_operators_path }}" - - CI-csi-orange-openshift_4_7: - name: "orange / Deploy openshift v4.7" + CI-csi-demo-pr: + name: "create demo pr in forked community-operators repo" needs: prepare_env runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: Checkout code uses: actions/checkout@v2 - - name: update_csv + - name: create demo PR run: | - build/ci/update_csv.sh + build/ci/create_demo_pr.sh env: + github_csiblock_community_operators_repository: "csiblock/community-operators" + community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" + community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" + github_token: ${{ secrets.CSIBLOCK_GITHUB_TOKEN }} current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} csv_file: ${{ needs.prepare_env.outputs.csv_file }} repository_path: ${{ needs.prepare_env.outputs.repository_path }} - - name: Operator test + - name: clean demo pr + if: always() run: | - echo "orange_v4.7 ${{ needs.prepare_env.outputs.community_operators_path }}" - build/ci/op-test.sh orange_v4.7 "${{ needs.prepare_env.outputs.community_operators_path }}" - - CI-csi-orange-openshift_4_8: - name: "orange / Deploy openshift v4.8" - needs: prepare_env - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: update_csv - run: | - build/ci/update_csv.sh + bash build/ci/clean_demo_pr.sh env: - current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} - wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} - csv_file: ${{ needs.prepare_env.outputs.csv_file }} - repository_path: ${{ needs.prepare_env.outputs.repository_path }} - - name: Operator test - run: | - echo "orange_v4.8 ${{ needs.prepare_env.outputs.community_operators_path }}" - build/ci/op-test.sh orange_v4.8 "${{ needs.prepare_env.outputs.community_operators_path }}" + github_csiblock_community_operators_repository: "csiblock/community-operators" + community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" + community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" diff --git a/build/ci/clean_demo_pr.sh b/build/ci/clean_demo_pr.sh new file mode 100755 index 000000000..cb354958a --- /dev/null +++ b/build/ci/clean_demo_pr.sh @@ -0,0 +1,42 @@ +#!/bin/bash -xe +set +o pipefail + +export failed_k8s_checks=false +export failed_openshift_checks=false + +export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_kubernetes_branch` +if [[ "$repo_pr" == *"$community_operators_kubernetes_branch"* ]]; then + sleep 5 + while [ "$(gh pr checks $community_operators_kubernetes_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do + sleep 5 + echo "there are still checks that are running..." + done + if [ "$(gh pr checks $community_operators_kubernetes_branch --repo $github_csiblock_community_operators_repository | grep -i fail | wc -l)" -gt 0 ] + then + export failed_k8s_checks=true + fi + echo "The k8s checks:" + gh pr checks $community_operators_kubernetes_branch --repo $github_csiblock_community_operators_repository + gh pr close $community_operators_kubernetes_branch --delete-branch --repo $github_csiblock_community_operators_repository +fi + +export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_openshift_branch` +if [[ "$repo_pr" == *"$community_operators_openshift_branch"* ]]; then + sleep 5 + while [ "$(gh pr checks $community_operators_openshift_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do + sleep 5 + echo "there are still checks that are running..." + done + if [ "$(gh pr checks $community_operators_openshift_branch --repo $github_csiblock_community_operators_repository | grep -i fail | wc -l)" -gt 0 ] + then + export failed_openshift_checks=true + fi + echo "The Openshift checks:" + gh pr checks $community_operators_openshift_branch --repo $github_csiblock_community_operators_repository + gh pr close $community_operators_openshift_branch --delete-branch --repo $github_csiblock_community_operators_repository +fi + +if [ $failed_k8s_checks == "true" ] || [ $failed_openshift_checks == "true" ] +then + exit 1 +fi diff --git a/build/ci/create_demo_pr.sh b/build/ci/create_demo_pr.sh new file mode 100755 index 000000000..da5976038 --- /dev/null +++ b/build/ci/create_demo_pr.sh @@ -0,0 +1,44 @@ +#!/bin/bash -xe +set +o pipefail + +sed -i "s+$current_operator_image+$wanted_operator_image+g" $csv_file + +# install gh command +curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null +sudo apt update +sudo apt install gh + +echo $github_token > mytoken.txt +gh auth login --with-token < mytoken.txt +gh repo fork operator-framework/community-operators --clone community-operators-fork +cd community-operators-fork +git remote set-url origin https://csiblock:$github_token@github.com/csiblock/community-operators.git +git fetch upstream +git rebase upstream/master +git push origin master --force +git config --global user.email csi.block1@il.ibm.com +git config --global user.name csiblock + +export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_kubernetes_branch` +if [[ "$repo_pr" == *"$community_operators_kubernetes_branch"* ]]; then + gh pr close $community_operators_kubernetes_branch --delete-branch --repo $github_csiblock_community_operators_repository +fi +git checkout -b $community_operators_kubernetes_branch +yes | cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/ $repository_path/community-operators-fork/upstream-community-operators/ +git add . +git commit --signoff -m "build number $github_build_number kubernetes" +git push origin $community_operators_kubernetes_branch +gh pr create --title "IBM Block CSI update to v1.6.0 kubernetes" --repo $github_csiblock_community_operators_repository --base master --head $community_operators_kubernetes_branch --body "pr check" + +export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_openshift_branch` +if [[ "$repo_pr" == *"$community_operators_openshift_branch"* ]]; then + gh pr close $community_operators_openshift_branch --delete-branch --repo $github_csiblock_community_operators_repository +fi +git checkout master +git checkout -b $community_operators_openshift_branch +yes | cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/ $repository_path/community-operators-fork/community-operators/ +git add . +git commit --signoff -m "build number $github_build_number openshift" +git push origin $community_operators_openshift_branch +gh pr create --title "IBM Block CSI update to v1.6.0 operator" --repo $github_csiblock_community_operators_repository --base master --head $community_operators_openshift_branch --body "pr check" diff --git a/build/ci/op-test.sh b/build/ci/op-test.sh deleted file mode 100755 index d5bc5800c..000000000 --- a/build/ci/op-test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -OP_TEST_SCRIPT_URL=${OP_TEST_SCRIPT_URL-"https://raw.githubusercontent.com/operator-framework/operator-test-playbooks/master/upstream/test/test.sh"} - -bash <(curl -sL $OP_TEST_SCRIPT_URL) $* -rc=$? -[[ $rc -eq 0 ]] && echo "rc=$rc" || { echo "Error: rc=$rc"; exit $rc; } diff --git a/build/ci/update_csv.sh b/build/ci/update_csv.sh deleted file mode 100755 index 8c720508a..000000000 --- a/build/ci/update_csv.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -xe -set +o pipefail - -sed -i "s+$current_operator_image+$wanted_operator_image+g" $csv_file -sed -i 's/.*minKubeVersion.*/ minKubeVersion: 1.19.0/' $csv_file - -mkdir $repository_path/upstream-community-operators/ -mkdir $repository_path/community-operators/ - -cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/ $repository_path/upstream-community-operators/ -cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/ $repository_path/community-operators/ diff --git a/categories.json b/categories.json deleted file mode 100644 index a0b2f49a7..000000000 --- a/categories.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "categories":[ - "AI/Machine Learning", - "Application Runtime", - "Big Data", - "Cloud Provider", - "Developer Tools", - "Database", - "Drivers and plugins", - "Integration & Delivery", - "Logging & Tracing", - "Modernization & Migration", - "Monitoring", - "Networking", - "OpenShift Optional", - "Security", - "Storage", - "Streaming & Messaging" - ] - } \ No newline at end of file diff --git a/deploy/olm-catalog/ibm-block-csi-operator-community/ci.yaml b/deploy/olm-catalog/ibm-block-csi-operator-community/ci.yaml deleted file mode 100644 index 337cb631a..000000000 --- a/deploy/olm-catalog/ibm-block-csi-operator-community/ci.yaml +++ /dev/null @@ -1,3 +0,0 @@ ---- -# Use `replaces-mode` or `semver-mode`. Once you switch to `semver-mode`, there is no easy way back. -updateGraph: replaces-mode \ No newline at end of file From 3ca39751c3d1c9360c4aae4ec3cccc9348bad706 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 10 Jun 2021 11:49:38 +0300 Subject: [PATCH 06/51] answer codereview comments --- .github/workflows/csi_operator_ci.yaml | 23 ++++------ build/ci/clean_demo_pr.sh | 61 +++++++++++++------------- build/ci/create_demo_pr.sh | 47 ++++++++++---------- build/ci/prepare_env.sh | 10 ++--- 4 files changed, 65 insertions(+), 76 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 8d9d001f0..307eb83c9 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -1,8 +1,9 @@ name: Community Operators test on: - push: + pull_request: branches: - - task/CSI-2892_cert_community_operator_hub + - develop + - master jobs: operator_image_build: name: "set env" @@ -11,7 +12,7 @@ jobs: - name: Checkout code uses: actions/checkout@v2 outputs: - operator_image: "csiblock1/ibm-block-csi-operator-amd64:new" + operator_image: "ibmcom/ibm-block-csi-operator:latest" prepare_env: name: "prepare_env" @@ -27,29 +28,26 @@ jobs: - name: Install dependencies run: python -m pip install --upgrade pip yq - uses: FranzDiebold/github-env-vars-action@v2 - - name: prepare_env + - name: Prepare_env id: environment_setup env: - operator_image: ${{ needs.operator_image_build.outputs.operator_image }} + operator_image_for_test: ${{ needs.operator_image_build.outputs.operator_image }} run: | build/ci/prepare_env.sh outputs: - current_operator_image: "${{ steps.environment_setup.outputs.current_operator_image }}" - wanted_operator_image: "${{ steps.environment_setup.outputs.wanted_operator_image }}" - csv_file: "${{ steps.environment_setup.outputs.csv_file }}" community_operators_path: "${{ steps.environment_setup.outputs.community_operators_path }}" upstream_community_operators_path: "${{ steps.environment_setup.outputs.upstream_community_operators_path }}" repository_path: "${{ steps.environment_setup.outputs.repository_path }}" CI-csi-demo-pr: - name: "create demo pr in forked community-operators repo" + name: "community-operators fork checks" needs: prepare_env runs-on: ubuntu-latest timeout-minutes: 30 steps: - name: Checkout code uses: actions/checkout@v2 - - name: create demo PR + - name: Create demo PR run: | build/ci/create_demo_pr.sh env: @@ -57,11 +55,8 @@ jobs: community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" github_token: ${{ secrets.CSIBLOCK_GITHUB_TOKEN }} - current_operator_image: ${{ needs.prepare_env.outputs.current_operator_image }} - wanted_operator_image: ${{ needs.prepare_env.outputs.wanted_operator_image }} - csv_file: ${{ needs.prepare_env.outputs.csv_file }} repository_path: ${{ needs.prepare_env.outputs.repository_path }} - - name: clean demo pr + - name: Clean demo PR if: always() run: | bash build/ci/clean_demo_pr.sh diff --git a/build/ci/clean_demo_pr.sh b/build/ci/clean_demo_pr.sh index cb354958a..169fd38c7 100755 --- a/build/ci/clean_demo_pr.sh +++ b/build/ci/clean_demo_pr.sh @@ -1,42 +1,41 @@ #!/bin/bash -xe set +o pipefail -export failed_k8s_checks=false -export failed_openshift_checks=false +export passed_k8s_checks=true +export passed_openshift_checks=true -export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_kubernetes_branch` -if [[ "$repo_pr" == *"$community_operators_kubernetes_branch"* ]]; then - sleep 5 - while [ "$(gh pr checks $community_operators_kubernetes_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do +wait_for_checks_to_complete(){ + community_operators_branch=$1 + all_checks_passed=true + export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` + if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then sleep 5 - echo "there are still checks that are running..." - done - if [ "$(gh pr checks $community_operators_kubernetes_branch --repo $github_csiblock_community_operators_repository | grep -i fail | wc -l)" -gt 0 ] - then - export failed_k8s_checks=true + while [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do + sleep 5 + done + if [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i fail | wc -l)" -gt 0 ] + then + export all_checks_passed=false + fi + echo "$all_checks_passed" fi - echo "The k8s checks:" - gh pr checks $community_operators_kubernetes_branch --repo $github_csiblock_community_operators_repository - gh pr close $community_operators_kubernetes_branch --delete-branch --repo $github_csiblock_community_operators_repository -fi +} -export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_openshift_branch` -if [[ "$repo_pr" == *"$community_operators_openshift_branch"* ]]; then - sleep 5 - while [ "$(gh pr checks $community_operators_openshift_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do - sleep 5 - echo "there are still checks that are running..." - done - if [ "$(gh pr checks $community_operators_openshift_branch --repo $github_csiblock_community_operators_repository | grep -i fail | wc -l)" -gt 0 ] - then - export failed_openshift_checks=true - fi - echo "The Openshift checks:" - gh pr checks $community_operators_openshift_branch --repo $github_csiblock_community_operators_repository - gh pr close $community_operators_openshift_branch --delete-branch --repo $github_csiblock_community_operators_repository -fi +print_checks_and_delete_pr(){ + community_operators_branch=$1 + cluster_kind=$2 + echo "The $cluster_kind checks:" + gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository + gh pr close $community_operators_branch --delete-branch --repo $github_csiblock_community_operators_repository +} + +passed_k8s_checks=$(wait_for_checks_to_complete $community_operators_kubernetes_branch) +print_checks_and_delete_pr $community_operators_kubernetes_branch 'kubernetes' +passed_openshift_checks=$(wait_for_checks_to_complete $community_operators_openshift_branch) +print_checks_and_delete_pr $community_operators_openshift_branch 'openshift' -if [ $failed_k8s_checks == "true" ] || [ $failed_openshift_checks == "true" ] +if [ $passed_k8s_checks == "false" ] || [ $passed_openshift_checks == "false" ] then + echo "some test failed :(" exit 1 fi diff --git a/build/ci/create_demo_pr.sh b/build/ci/create_demo_pr.sh index da5976038..8b0cd56ca 100755 --- a/build/ci/create_demo_pr.sh +++ b/build/ci/create_demo_pr.sh @@ -6,11 +6,11 @@ sed -i "s+$current_operator_image+$wanted_operator_image+g" $csv_file # install gh command curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null -sudo apt update -sudo apt install gh +sudo apt-get update +sudo apt-get install gh -echo $github_token > mytoken.txt -gh auth login --with-token < mytoken.txt +echo $github_token > github_token.txt +gh auth login --with-token < github_token.txt gh repo fork operator-framework/community-operators --clone community-operators-fork cd community-operators-fork git remote set-url origin https://csiblock:$github_token@github.com/csiblock/community-operators.git @@ -20,25 +20,22 @@ git push origin master --force git config --global user.email csi.block1@il.ibm.com git config --global user.name csiblock -export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_kubernetes_branch` -if [[ "$repo_pr" == *"$community_operators_kubernetes_branch"* ]]; then - gh pr close $community_operators_kubernetes_branch --delete-branch --repo $github_csiblock_community_operators_repository -fi -git checkout -b $community_operators_kubernetes_branch -yes | cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/ $repository_path/community-operators-fork/upstream-community-operators/ -git add . -git commit --signoff -m "build number $github_build_number kubernetes" -git push origin $community_operators_kubernetes_branch -gh pr create --title "IBM Block CSI update to v1.6.0 kubernetes" --repo $github_csiblock_community_operators_repository --base master --head $community_operators_kubernetes_branch --body "pr check" +create_demo_pr(){ + community_operators_branch=$1 + dest_path=$2 + cluster_kind=$3 + export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` + if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then + gh pr close $community_operators_branch --delete-branch --repo $github_csiblock_community_operators_repository + fi + git checkout master + git checkout -b $community_operators_branch + yes | cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/ $dest_path + git add . + git commit --signoff -m "build number $github_build_number $cluster_kind" + git push origin $community_operators_branch + gh pr create --title "IBM Block CSI update $cluster_kind" --repo $github_csiblock_community_operators_repository --base master --head $community_operators_branch --body "pr check" +} -export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_openshift_branch` -if [[ "$repo_pr" == *"$community_operators_openshift_branch"* ]]; then - gh pr close $community_operators_openshift_branch --delete-branch --repo $github_csiblock_community_operators_repository -fi -git checkout master -git checkout -b $community_operators_openshift_branch -yes | cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/ $repository_path/community-operators-fork/community-operators/ -git add . -git commit --signoff -m "build number $github_build_number openshift" -git push origin $community_operators_openshift_branch -gh pr create --title "IBM Block CSI update to v1.6.0 operator" --repo $github_csiblock_community_operators_repository --base master --head $community_operators_openshift_branch --body "pr check" +create_demo_pr $community_operators_kubernetes_branch "upstream-community-operators/" "kubernetes" +create_demo_pr $community_operators_openshift_branch "community-operators/" "openshift" diff --git a/build/ci/prepare_env.sh b/build/ci/prepare_env.sh index 56651cf67..9e3f6b269 100755 --- a/build/ci/prepare_env.sh +++ b/build/ci/prepare_env.sh @@ -6,15 +6,13 @@ export csv_version=`echo ${csv_version//ibm-block-csi-operator.v}` export csv_version=`echo ${csv_version//\"}` export repository_path=~/work/$CI_REPOSITORY_NAME/$CI_REPOSITORY_NAME export csv_file=$repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$csv_version/ibm-block-csi-operator.v$csv_version.clusterserviceversion.yaml -export wanted_image=$operator_image -export current_image=`yq .metadata.annotations.containerImage $csv_file` -export current_image=`echo ${current_image//\"}` +export operator_image_in_branch=`yq .metadata.annotations.containerImage $csv_file` +export operator_image_in_branch=`echo ${operator_image_in_branch//\"}` export upstream_community_operators_path=upstream-community-operators/ibm-block-csi-operator-community/$csv_version export community_operators_path=community-operators/ibm-block-csi-operator-community/$csv_version -echo "::set-output name=current_operator_image::${current_image}" -echo "::set-output name=wanted_operator_image::${wanted_image}" -echo "::set-output name=csv_file::${csv_file}" +sed -i "s+$operator_image_for_test+$operator_image_in_branch+g" $csv_file + echo "::set-output name=upstream_community_operators_path::${upstream_community_operators_path}" echo "::set-output name=community_operators_path::${community_operators_path}" echo "::set-output name=repository_path::${repository_path}" From b267d6d6ba86fb2608796f43db0cc7d629ab3c6a Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 10 Jun 2021 11:56:10 +0300 Subject: [PATCH 07/51] remove sed from create_demo_pr.sh --- build/ci/create_demo_pr.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/ci/create_demo_pr.sh b/build/ci/create_demo_pr.sh index 8b0cd56ca..3bb63488c 100755 --- a/build/ci/create_demo_pr.sh +++ b/build/ci/create_demo_pr.sh @@ -1,8 +1,6 @@ #!/bin/bash -xe set +o pipefail -sed -i "s+$current_operator_image+$wanted_operator_image+g" $csv_file - # install gh command curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null From f9fdda3d11630d1a20d0b8dde7405b161e0bb11b Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 10 Jun 2021 13:34:31 +0300 Subject: [PATCH 08/51] move back the sed command to create_pr --- build/ci/create_demo_pr.sh | 2 ++ build/ci/prepare_env.sh | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/ci/create_demo_pr.sh b/build/ci/create_demo_pr.sh index 3bb63488c..5c9d2dd8f 100755 --- a/build/ci/create_demo_pr.sh +++ b/build/ci/create_demo_pr.sh @@ -1,6 +1,8 @@ #!/bin/bash -xe set +o pipefail +sed -i "s+$operator_image_for_test+$operator_image_in_branch+g" $csv_file + # install gh command curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null diff --git a/build/ci/prepare_env.sh b/build/ci/prepare_env.sh index 9e3f6b269..c4cd8cf7d 100755 --- a/build/ci/prepare_env.sh +++ b/build/ci/prepare_env.sh @@ -11,8 +11,6 @@ export operator_image_in_branch=`echo ${operator_image_in_branch//\"}` export upstream_community_operators_path=upstream-community-operators/ibm-block-csi-operator-community/$csv_version export community_operators_path=community-operators/ibm-block-csi-operator-community/$csv_version -sed -i "s+$operator_image_for_test+$operator_image_in_branch+g" $csv_file - echo "::set-output name=upstream_community_operators_path::${upstream_community_operators_path}" echo "::set-output name=community_operators_path::${community_operators_path}" echo "::set-output name=repository_path::${repository_path}" From 218a95a9ade147e4497a00f4de9adcf34fe45e29 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 10 Jun 2021 13:40:07 +0300 Subject: [PATCH 09/51] pass operator_image_in_branch variable --- .github/workflows/csi_operator_ci.yaml | 2 ++ build/ci/prepare_env.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 307eb83c9..343074ad5 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -38,6 +38,7 @@ jobs: community_operators_path: "${{ steps.environment_setup.outputs.community_operators_path }}" upstream_community_operators_path: "${{ steps.environment_setup.outputs.upstream_community_operators_path }}" repository_path: "${{ steps.environment_setup.outputs.repository_path }}" + operator_image_in_branch: "${{ steps.environment_setup.outputs.operator_image_in_branch }}" CI-csi-demo-pr: name: "community-operators fork checks" @@ -56,6 +57,7 @@ jobs: community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" github_token: ${{ secrets.CSIBLOCK_GITHUB_TOKEN }} repository_path: ${{ needs.prepare_env.outputs.repository_path }} + operator_image_in_branch: ${{ needs.prepare_env.outputs.operator_image_in_branch }} - name: Clean demo PR if: always() run: | diff --git a/build/ci/prepare_env.sh b/build/ci/prepare_env.sh index c4cd8cf7d..e61909c16 100755 --- a/build/ci/prepare_env.sh +++ b/build/ci/prepare_env.sh @@ -11,6 +11,7 @@ export operator_image_in_branch=`echo ${operator_image_in_branch//\"}` export upstream_community_operators_path=upstream-community-operators/ibm-block-csi-operator-community/$csv_version export community_operators_path=community-operators/ibm-block-csi-operator-community/$csv_version +echo "::set-output name=operator_image_in_branch::${operator_image_in_branch}" echo "::set-output name=upstream_community_operators_path::${upstream_community_operators_path}" echo "::set-output name=community_operators_path::${community_operators_path}" echo "::set-output name=repository_path::${repository_path}" From 975fc661fa754d69f204322dcc152a1dddc97512 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 10 Jun 2021 13:56:39 +0300 Subject: [PATCH 10/51] passed operator_image_for_test to CI-csi-demo-pr --- .github/workflows/csi_operator_ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 343074ad5..2073b3608 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -30,8 +30,6 @@ jobs: - uses: FranzDiebold/github-env-vars-action@v2 - name: Prepare_env id: environment_setup - env: - operator_image_for_test: ${{ needs.operator_image_build.outputs.operator_image }} run: | build/ci/prepare_env.sh outputs: @@ -39,6 +37,7 @@ jobs: upstream_community_operators_path: "${{ steps.environment_setup.outputs.upstream_community_operators_path }}" repository_path: "${{ steps.environment_setup.outputs.repository_path }}" operator_image_in_branch: "${{ steps.environment_setup.outputs.operator_image_in_branch }}" + operator_image_for_test: ${{ needs.operator_image_build.outputs.operator_image }} CI-csi-demo-pr: name: "community-operators fork checks" @@ -58,6 +57,7 @@ jobs: github_token: ${{ secrets.CSIBLOCK_GITHUB_TOKEN }} repository_path: ${{ needs.prepare_env.outputs.repository_path }} operator_image_in_branch: ${{ needs.prepare_env.outputs.operator_image_in_branch }} + operator_image_for_test: ${{ needs.prepare_env.outputs.operator_image_for_test }} - name: Clean demo PR if: always() run: | From 362c38fb5be889ac4867afa5056d41c5d8938bef Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 10 Jun 2021 14:11:20 +0300 Subject: [PATCH 11/51] move csv_file variable to CI-csi-demo-pr --- .github/workflows/csi_operator_ci.yaml | 2 ++ build/ci/prepare_env.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 2073b3608..142879195 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -38,6 +38,7 @@ jobs: repository_path: "${{ steps.environment_setup.outputs.repository_path }}" operator_image_in_branch: "${{ steps.environment_setup.outputs.operator_image_in_branch }}" operator_image_for_test: ${{ needs.operator_image_build.outputs.operator_image }} + csv_file: ${{ needs.operator_image_build.outputs.csv_file }} CI-csi-demo-pr: name: "community-operators fork checks" @@ -58,6 +59,7 @@ jobs: repository_path: ${{ needs.prepare_env.outputs.repository_path }} operator_image_in_branch: ${{ needs.prepare_env.outputs.operator_image_in_branch }} operator_image_for_test: ${{ needs.prepare_env.outputs.operator_image_for_test }} + csv_file: ${{ needs.prepare_env.outputs.csv_file }} - name: Clean demo PR if: always() run: | diff --git a/build/ci/prepare_env.sh b/build/ci/prepare_env.sh index e61909c16..cfa2d6a9e 100755 --- a/build/ci/prepare_env.sh +++ b/build/ci/prepare_env.sh @@ -15,3 +15,4 @@ echo "::set-output name=operator_image_in_branch::${operator_image_in_branch}" echo "::set-output name=upstream_community_operators_path::${upstream_community_operators_path}" echo "::set-output name=community_operators_path::${community_operators_path}" echo "::set-output name=repository_path::${repository_path}" +echo "::set-output name=repository_path::${csv_file}" From 3bb0c930ed3f548611c8c3b671152f723258b0fb Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 10 Jun 2021 14:50:49 +0300 Subject: [PATCH 12/51] add check and clean stages --- .github/workflows/csi_operator_ci.yaml | 13 +++++-- .../check_all_demo_pr_checks_were_passed.sh | 31 ++++++++++++++++ build/ci/clean_demo_pr.sh | 35 +++---------------- build/ci/create_demo_pr.sh | 2 +- build/ci/prepare_env.sh | 2 +- 5 files changed, 48 insertions(+), 35 deletions(-) create mode 100755 build/ci/check_all_demo_pr_checks_were_passed.sh diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 142879195..f3dc95421 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -12,7 +12,7 @@ jobs: - name: Checkout code uses: actions/checkout@v2 outputs: - operator_image: "ibmcom/ibm-block-csi-operator:latest" + operator_image_for_test: "ibmcom/ibm-block-csi-operator:latest" prepare_env: name: "prepare_env" @@ -38,7 +38,7 @@ jobs: repository_path: "${{ steps.environment_setup.outputs.repository_path }}" operator_image_in_branch: "${{ steps.environment_setup.outputs.operator_image_in_branch }}" operator_image_for_test: ${{ needs.operator_image_build.outputs.operator_image }} - csv_file: ${{ needs.operator_image_build.outputs.csv_file }} + csv_file: ${{ steps.environment_setup.outputs.csv_file }} CI-csi-demo-pr: name: "community-operators fork checks" @@ -60,10 +60,17 @@ jobs: operator_image_in_branch: ${{ needs.prepare_env.outputs.operator_image_in_branch }} operator_image_for_test: ${{ needs.prepare_env.outputs.operator_image_for_test }} csv_file: ${{ needs.prepare_env.outputs.csv_file }} + - name: Check if all the demo PR checks were passed + run: | + build/ci/check_all_demo_pr_checks_were_passed.sh + env: + github_csiblock_community_operators_repository: "csiblock/community-operators" + community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" + community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" - name: Clean demo PR if: always() run: | - bash build/ci/clean_demo_pr.sh + build/ci/clean_demo_pr.sh env: github_csiblock_community_operators_repository: "csiblock/community-operators" community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" diff --git a/build/ci/check_all_demo_pr_checks_were_passed.sh b/build/ci/check_all_demo_pr_checks_were_passed.sh new file mode 100755 index 000000000..76c9eb1c8 --- /dev/null +++ b/build/ci/check_all_demo_pr_checks_were_passed.sh @@ -0,0 +1,31 @@ +#!/bin/bash -xe +set +o pipefail + +export passed_k8s_checks=true +export passed_openshift_checks=true + +wait_for_checks_to_complete(){ + community_operators_branch=$1 + all_checks_passed=true + export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` + if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then + sleep 5 + while [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do + sleep 5 + done + if [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i fail | wc -l)" -gt 0 ] + then + export all_checks_passed=false + fi + echo "$all_checks_passed" + fi +} + +passed_k8s_checks=$(wait_for_checks_to_complete $community_operators_kubernetes_branch) +passed_openshift_checks=$(wait_for_checks_to_complete $community_operators_openshift_branch) + +if [ $passed_k8s_checks == "false" ] || [ $passed_openshift_checks == "false" ] +then + echo "some test failed :(" + exit 1 +fi diff --git a/build/ci/clean_demo_pr.sh b/build/ci/clean_demo_pr.sh index 169fd38c7..348085964 100755 --- a/build/ci/clean_demo_pr.sh +++ b/build/ci/clean_demo_pr.sh @@ -1,41 +1,16 @@ #!/bin/bash -xe set +o pipefail -export passed_k8s_checks=true -export passed_openshift_checks=true - -wait_for_checks_to_complete(){ +print_checks_and_delete_pr(){ community_operators_branch=$1 - all_checks_passed=true + cluster_kind=$2 export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then - sleep 5 - while [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do - sleep 5 - done - if [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i fail | wc -l)" -gt 0 ] - then - export all_checks_passed=false - fi - echo "$all_checks_passed" + echo "The $cluster_kind checks:" + gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository + gh pr close $community_operators_branch --delete-branch --repo $github_csiblock_community_operators_repository fi } -print_checks_and_delete_pr(){ - community_operators_branch=$1 - cluster_kind=$2 - echo "The $cluster_kind checks:" - gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository - gh pr close $community_operators_branch --delete-branch --repo $github_csiblock_community_operators_repository -} - -passed_k8s_checks=$(wait_for_checks_to_complete $community_operators_kubernetes_branch) print_checks_and_delete_pr $community_operators_kubernetes_branch 'kubernetes' -passed_openshift_checks=$(wait_for_checks_to_complete $community_operators_openshift_branch) print_checks_and_delete_pr $community_operators_openshift_branch 'openshift' - -if [ $passed_k8s_checks == "false" ] || [ $passed_openshift_checks == "false" ] -then - echo "some test failed :(" - exit 1 -fi diff --git a/build/ci/create_demo_pr.sh b/build/ci/create_demo_pr.sh index 5c9d2dd8f..19ad339b8 100755 --- a/build/ci/create_demo_pr.sh +++ b/build/ci/create_demo_pr.sh @@ -1,7 +1,7 @@ #!/bin/bash -xe set +o pipefail -sed -i "s+$operator_image_for_test+$operator_image_in_branch+g" $csv_file +sed -i "s+$operator_image_in_branch+$operator_image_for_test+g" $csv_file # install gh command curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg diff --git a/build/ci/prepare_env.sh b/build/ci/prepare_env.sh index cfa2d6a9e..c93a47d10 100755 --- a/build/ci/prepare_env.sh +++ b/build/ci/prepare_env.sh @@ -15,4 +15,4 @@ echo "::set-output name=operator_image_in_branch::${operator_image_in_branch}" echo "::set-output name=upstream_community_operators_path::${upstream_community_operators_path}" echo "::set-output name=community_operators_path::${community_operators_path}" echo "::set-output name=repository_path::${repository_path}" -echo "::set-output name=repository_path::${csv_file}" +echo "::set-output name=csv_file::${csv_file}" From 89e065e9d332685dbb5a42419b019526467ab990 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 10 Jun 2021 16:40:13 +0300 Subject: [PATCH 13/51] changed files name --- .github/workflows/csi_operator_ci.yaml | 8 ++++++-- ...o_pr_checks_were_passed.sh => check_demo_pr_checks.sh} | 0 2 files changed, 6 insertions(+), 2 deletions(-) rename build/ci/{check_all_demo_pr_checks_were_passed.sh => check_demo_pr_checks.sh} (100%) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index f3dc95421..ac8f4c9b2 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -1,5 +1,9 @@ name: Community Operators test on: + push: + branches: + - develop + - master pull_request: branches: - develop @@ -37,7 +41,7 @@ jobs: upstream_community_operators_path: "${{ steps.environment_setup.outputs.upstream_community_operators_path }}" repository_path: "${{ steps.environment_setup.outputs.repository_path }}" operator_image_in_branch: "${{ steps.environment_setup.outputs.operator_image_in_branch }}" - operator_image_for_test: ${{ needs.operator_image_build.outputs.operator_image }} + operator_image_for_test: ${{ needs.operator_image_build.outputs.operator_image_for_test }} csv_file: ${{ steps.environment_setup.outputs.csv_file }} CI-csi-demo-pr: @@ -62,7 +66,7 @@ jobs: csv_file: ${{ needs.prepare_env.outputs.csv_file }} - name: Check if all the demo PR checks were passed run: | - build/ci/check_all_demo_pr_checks_were_passed.sh + build/ci/check_demo_pr_checks.sh env: github_csiblock_community_operators_repository: "csiblock/community-operators" community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" diff --git a/build/ci/check_all_demo_pr_checks_were_passed.sh b/build/ci/check_demo_pr_checks.sh similarity index 100% rename from build/ci/check_all_demo_pr_checks_were_passed.sh rename to build/ci/check_demo_pr_checks.sh From f3959c694d3baa49378e55007ab172d27a6df530 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 10 Jun 2021 17:48:22 +0300 Subject: [PATCH 14/51] add debug in the ci --- .github/workflows/csi_operator_ci.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index ac8f4c9b2..8f702d2d8 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -64,9 +64,11 @@ jobs: operator_image_in_branch: ${{ needs.prepare_env.outputs.operator_image_in_branch }} operator_image_for_test: ${{ needs.prepare_env.outputs.operator_image_for_test }} csv_file: ${{ needs.prepare_env.outputs.csv_file }} - - name: Check if all the demo PR checks were passed - run: | - build/ci/check_demo_pr_checks.sh + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + #- name: Check if all the demo PR checks were passed + # run: | + # build/ci/check_demo_pr_checks.sh env: github_csiblock_community_operators_repository: "csiblock/community-operators" community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" From c57a0a6e6882ece61232deddd7ced0fc047c0daa Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 10 Jun 2021 18:16:23 +0300 Subject: [PATCH 15/51] remove debug --- .github/workflows/csi_operator_ci.yaml | 8 +++----- build/ci/check_demo_pr_checks.sh | 2 +- build/ci/clean_demo_pr.sh | 2 ++ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 8f702d2d8..ac8f4c9b2 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -64,11 +64,9 @@ jobs: operator_image_in_branch: ${{ needs.prepare_env.outputs.operator_image_in_branch }} operator_image_for_test: ${{ needs.prepare_env.outputs.operator_image_for_test }} csv_file: ${{ needs.prepare_env.outputs.csv_file }} - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - #- name: Check if all the demo PR checks were passed - # run: | - # build/ci/check_demo_pr_checks.sh + - name: Check if all the demo PR checks were passed + run: | + build/ci/check_demo_pr_checks.sh env: github_csiblock_community_operators_repository: "csiblock/community-operators" community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" diff --git a/build/ci/check_demo_pr_checks.sh b/build/ci/check_demo_pr_checks.sh index 76c9eb1c8..edca6cbc0 100755 --- a/build/ci/check_demo_pr_checks.sh +++ b/build/ci/check_demo_pr_checks.sh @@ -11,7 +11,7 @@ wait_for_checks_to_complete(){ if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then sleep 5 while [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do - sleep 5 + sleep 15 done if [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i fail | wc -l)" -gt 0 ] then diff --git a/build/ci/clean_demo_pr.sh b/build/ci/clean_demo_pr.sh index 348085964..e975ba5c2 100755 --- a/build/ci/clean_demo_pr.sh +++ b/build/ci/clean_demo_pr.sh @@ -1,6 +1,8 @@ #!/bin/bash -xe set +o pipefail +sleep 20 + print_checks_and_delete_pr(){ community_operators_branch=$1 cluster_kind=$2 From 4181fe0ac4d2fd878376d8cecce7cea196e67b5c Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Sun, 13 Jun 2021 14:39:24 +0300 Subject: [PATCH 16/51] add summary check --- .github/workflows/csi_operator_ci.yaml | 2 +- build/ci/check_demo_pr_checks.sh | 15 +++++++++++++++ build/ci/clean_demo_pr.sh | 2 -- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index ac8f4c9b2..10c7f7f3d 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -19,7 +19,7 @@ jobs: operator_image_for_test: "ibmcom/ibm-block-csi-operator:latest" prepare_env: - name: "prepare_env" + name: "prepare env" needs: operator_image_build runs-on: ubuntu-latest steps: diff --git a/build/ci/check_demo_pr_checks.sh b/build/ci/check_demo_pr_checks.sh index edca6cbc0..425c07892 100755 --- a/build/ci/check_demo_pr_checks.sh +++ b/build/ci/check_demo_pr_checks.sh @@ -13,6 +13,21 @@ wait_for_checks_to_complete(){ while [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do sleep 15 done + summary_counter=0 + export search_summary_test=`gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i summary` + while [ "$search_summary_test" != *"pending"* ] && [ $summary_counter -lt 20 ]; do + sleep 1 + export search_summary_test=`gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i summary` + ((summary_counter=summary_counter+1)) + if [[ "$search_summary_test" == *"pass"* ]]; then + summary_counter=20 + fi + done + if [[ $summary_counter -lt 20 ]]; then + while [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do + sleep 5 + done + fi if [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i fail | wc -l)" -gt 0 ] then export all_checks_passed=false diff --git a/build/ci/clean_demo_pr.sh b/build/ci/clean_demo_pr.sh index e975ba5c2..348085964 100755 --- a/build/ci/clean_demo_pr.sh +++ b/build/ci/clean_demo_pr.sh @@ -1,8 +1,6 @@ #!/bin/bash -xe set +o pipefail -sleep 20 - print_checks_and_delete_pr(){ community_operators_branch=$1 cluster_kind=$2 From 466059a48123ea3ccb819a8a2717f1f942eb52ff Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Sun, 13 Jun 2021 15:47:00 +0300 Subject: [PATCH 17/51] add install_ci_dependencies.sh file --- .github/workflows/csi_operator_ci.yaml | 3 +++ build/ci/create_demo_pr.sh | 6 ------ build/ci/install_ci_dependencies.sh | 8 ++++++++ 3 files changed, 11 insertions(+), 6 deletions(-) create mode 100755 build/ci/install_ci_dependencies.sh diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 10c7f7f3d..52b986774 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -31,6 +31,9 @@ jobs: python-version: '3.x' - name: Install dependencies run: python -m pip install --upgrade pip yq + - name: install apt dependencies + run: | + build/ci/install_ci_dependencies.sh - uses: FranzDiebold/github-env-vars-action@v2 - name: Prepare_env id: environment_setup diff --git a/build/ci/create_demo_pr.sh b/build/ci/create_demo_pr.sh index 19ad339b8..df328b637 100755 --- a/build/ci/create_demo_pr.sh +++ b/build/ci/create_demo_pr.sh @@ -3,12 +3,6 @@ set +o pipefail sed -i "s+$operator_image_in_branch+$operator_image_for_test+g" $csv_file -# install gh command -curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg -echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null -sudo apt-get update -sudo apt-get install gh - echo $github_token > github_token.txt gh auth login --with-token < github_token.txt gh repo fork operator-framework/community-operators --clone community-operators-fork diff --git a/build/ci/install_ci_dependencies.sh b/build/ci/install_ci_dependencies.sh new file mode 100755 index 000000000..c86bc3ef3 --- /dev/null +++ b/build/ci/install_ci_dependencies.sh @@ -0,0 +1,8 @@ +#!/bin/bash -xe +set +o pipefail + +# install gh command +curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null +sudo apt-get update +sudo apt-get install gh \ No newline at end of file From aefc113782992fc2309b272c0e64581e243bde41 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Sun, 13 Jun 2021 16:03:44 +0300 Subject: [PATCH 18/51] change fail check --- .github/workflows/csi_operator_ci.yaml | 2 +- build/ci/check_demo_pr_checks.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 52b986774..fa9dbec7e 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -67,7 +67,7 @@ jobs: operator_image_in_branch: ${{ needs.prepare_env.outputs.operator_image_in_branch }} operator_image_for_test: ${{ needs.prepare_env.outputs.operator_image_for_test }} csv_file: ${{ needs.prepare_env.outputs.csv_file }} - - name: Check if all the demo PR checks were passed + - name: Assert all demo PR checks passed run: | build/ci/check_demo_pr_checks.sh env: diff --git a/build/ci/check_demo_pr_checks.sh b/build/ci/check_demo_pr_checks.sh index 425c07892..abb0c762e 100755 --- a/build/ci/check_demo_pr_checks.sh +++ b/build/ci/check_demo_pr_checks.sh @@ -28,7 +28,7 @@ wait_for_checks_to_complete(){ sleep 5 done fi - if [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i fail | wc -l)" -gt 0 ] + if [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -iv pass | wc -l)" -gt 0 ] then export all_checks_passed=false fi From 68c5583d61e1aba0b9d93593389d1a856f0ff836 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Sun, 13 Jun 2021 16:14:39 +0300 Subject: [PATCH 19/51] remove underscore --- .github/workflows/csi_operator_ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index fa9dbec7e..2948b9969 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -35,7 +35,7 @@ jobs: run: | build/ci/install_ci_dependencies.sh - uses: FranzDiebold/github-env-vars-action@v2 - - name: Prepare_env + - name: Prepare env id: environment_setup run: | build/ci/prepare_env.sh From 5fd4b1142cf41a4458501ca85d27004ee5e48108 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Sun, 13 Jun 2021 16:26:33 +0300 Subject: [PATCH 20/51] add new line in install_ci_dependencies.sh --- build/ci/install_ci_dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci/install_ci_dependencies.sh b/build/ci/install_ci_dependencies.sh index c86bc3ef3..5d2b1f50f 100755 --- a/build/ci/install_ci_dependencies.sh +++ b/build/ci/install_ci_dependencies.sh @@ -5,4 +5,4 @@ set +o pipefail curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null sudo apt-get update -sudo apt-get install gh \ No newline at end of file +sudo apt-get install gh From f58e6f493eeba993456e6786e97eb3cbb6c08c78 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Mon, 14 Jun 2021 09:43:04 +0300 Subject: [PATCH 21/51] create are_all_tests_were_passed function --- .github/workflows/csi_operator_ci.yaml | 2 -- build/ci/check_demo_pr_checks.sh | 30 ++++++++++++++++---------- build/ci/install_ci_dependencies.sh | 2 ++ 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 2948b9969..af6afec89 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -30,8 +30,6 @@ jobs: with: python-version: '3.x' - name: Install dependencies - run: python -m pip install --upgrade pip yq - - name: install apt dependencies run: | build/ci/install_ci_dependencies.sh - uses: FranzDiebold/github-env-vars-action@v2 diff --git a/build/ci/check_demo_pr_checks.sh b/build/ci/check_demo_pr_checks.sh index abb0c762e..a834abf7d 100755 --- a/build/ci/check_demo_pr_checks.sh +++ b/build/ci/check_demo_pr_checks.sh @@ -6,7 +6,7 @@ export passed_openshift_checks=true wait_for_checks_to_complete(){ community_operators_branch=$1 - all_checks_passed=true + is_summary_test_passed=false export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then sleep 5 @@ -15,29 +15,37 @@ wait_for_checks_to_complete(){ done summary_counter=0 export search_summary_test=`gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i summary` - while [ "$search_summary_test" != *"pending"* ] && [ $summary_counter -lt 20 ]; do + while [ "$search_summary_test" != *"pending"* ] && [ $summary_counter -lt 20 ] && [ $is_summary_test_passed == "false" ]; do sleep 1 export search_summary_test=`gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i summary` ((summary_counter=summary_counter+1)) if [[ "$search_summary_test" == *"pass"* ]]; then - summary_counter=20 + is_summary_test_passed=true fi done - if [[ $summary_counter -lt 20 ]]; then + if [[ $is_summary_test_passed == "false" ]]; then while [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do sleep 5 done fi - if [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -iv pass | wc -l)" -gt 0 ] - then - export all_checks_passed=false - fi - echo "$all_checks_passed" fi } -passed_k8s_checks=$(wait_for_checks_to_complete $community_operators_kubernetes_branch) -passed_openshift_checks=$(wait_for_checks_to_complete $community_operators_openshift_branch) +are_all_tests_were_passed(){ + community_operators_branch=$1 + all_checks_passed=false + if [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -iv pass | wc -l)" -eq 0 ] + then + export all_checks_passed=true + fi + echo "$all_checks_passed" +} + +wait_for_checks_to_complete $community_operators_kubernetes_branch +wait_for_checks_to_complete $community_operators_openshift_branch + +passed_k8s_checks=$(are_all_tests_were_passed $community_operators_kubernetes_branch) +passed_openshift_checks=$(are_all_tests_were_passed $community_operators_openshift_branch) if [ $passed_k8s_checks == "false" ] || [ $passed_openshift_checks == "false" ] then diff --git a/build/ci/install_ci_dependencies.sh b/build/ci/install_ci_dependencies.sh index 5d2b1f50f..abbae2b78 100755 --- a/build/ci/install_ci_dependencies.sh +++ b/build/ci/install_ci_dependencies.sh @@ -1,6 +1,8 @@ #!/bin/bash -xe set +o pipefail +python -m pip install --upgrade pip yq + # install gh command curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null From ca18f571cb5054089594d642a82bccd72dcccd33 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Mon, 14 Jun 2021 21:58:52 +0300 Subject: [PATCH 22/51] add some cache handle --- .github/workflows/csi_operator_ci.yaml | 59 +++++++++++++++---- build/ci/check_all_checks.sh | 24 ++++++++ build/ci/check_demo_pr_checks.sh | 54 ----------------- build/ci/create_demo_pr.sh | 9 ++- ...pendencies.sh => setup_ci_dependencies.sh} | 5 ++ build/ci/wait_demo_pr_checks.sh | 33 +++++++++++ 6 files changed, 115 insertions(+), 69 deletions(-) create mode 100755 build/ci/check_all_checks.sh delete mode 100755 build/ci/check_demo_pr_checks.sh rename build/ci/{install_ci_dependencies.sh => setup_ci_dependencies.sh} (76%) create mode 100755 build/ci/wait_demo_pr_checks.sh diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index af6afec89..e42c2437e 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -22,6 +22,10 @@ jobs: name: "prepare env" needs: operator_image_build runs-on: ubuntu-latest + env: + github_csiblock_community_operators_repository: "csiblock/community-operators" + community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" + community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" steps: - name: Checkout code uses: actions/checkout@v2 @@ -29,9 +33,19 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.x' - - name: Install dependencies + - name: Setup dependencies run: | - build/ci/install_ci_dependencies.sh + build/ci/setup_ci_dependencies.sh + - uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('dev-requirements.txt') }} + - name: Save gitconfig file + uses: actions/upload-artifact@v2 + with: + name: gitconfig-file + path: /home/runner/.gitconfig + retention-days: 1 - uses: FranzDiebold/github-env-vars-action@v2 - name: Prepare env id: environment_setup @@ -44,6 +58,7 @@ jobs: operator_image_in_branch: "${{ steps.environment_setup.outputs.operator_image_in_branch }}" operator_image_for_test: ${{ needs.operator_image_build.outputs.operator_image_for_test }} csv_file: ${{ steps.environment_setup.outputs.csv_file }} + github_build_number: ${{ github.run_number }} CI-csi-demo-pr: name: "community-operators fork checks" @@ -53,30 +68,50 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('dev-requirements.txt') }} + - name: Retrieve gitconfig file + uses: actions/download-artifact@v2 + with: + name: gitconfig-file + path: /home/runner - name: Create demo PR run: | build/ci/create_demo_pr.sh env: - github_csiblock_community_operators_repository: "csiblock/community-operators" - community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" - community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" + github_csiblock_community_operators_repository: ${{ env.github_csiblock_community_operators_repository }} + community_operators_kubernetes_branch: ${{ env.community_operators_kubernetes_branch }} + community_operators_openshift_branch: ${{ env.community_operators_openshift_branch }} github_token: ${{ secrets.CSIBLOCK_GITHUB_TOKEN }} repository_path: ${{ needs.prepare_env.outputs.repository_path }} operator_image_in_branch: ${{ needs.prepare_env.outputs.operator_image_in_branch }} operator_image_for_test: ${{ needs.prepare_env.outputs.operator_image_for_test }} csv_file: ${{ needs.prepare_env.outputs.csv_file }} + - name: Wait for all the checks to finish running + run: | + build/ci/wait_demo_pr_checks.sh + env: + github_csiblock_community_operators_repository: ${{ env.github_csiblock_community_operators_repository }} + community_operators_kubernetes_branch: ${{ env.community_operators_kubernetes_branch }} + community_operators_openshift_branch: ${{ env.community_operators_openshift_branch }} - name: Assert all demo PR checks passed run: | - build/ci/check_demo_pr_checks.sh + build/ci/check_all_checks.sh env: - github_csiblock_community_operators_repository: "csiblock/community-operators" - community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" - community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" + github_csiblock_community_operators_repository: ${{ env.github_csiblock_community_operators_repository }} + community_operators_kubernetes_branch: ${{ env.community_operators_kubernetes_branch }} + community_operators_openshift_branch: ${{ env.community_operators_openshift_branch }} - name: Clean demo PR if: always() run: | build/ci/clean_demo_pr.sh env: - github_csiblock_community_operators_repository: "csiblock/community-operators" - community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" - community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" + github_csiblock_community_operators_repository: ${{ env.github_csiblock_community_operators_repository }} + community_operators_kubernetes_branch: ${{ env.community_operators_kubernetes_branch }} + community_operators_openshift_branch: ${{ env.community_operators_openshift_branch }} diff --git a/build/ci/check_all_checks.sh b/build/ci/check_all_checks.sh new file mode 100755 index 000000000..8f21f556f --- /dev/null +++ b/build/ci/check_all_checks.sh @@ -0,0 +1,24 @@ +#!/bin/bash -xe +set +o pipefail + +export passed_k8s_checks=true +export passed_openshift_checks=true + +did_all_checks_pass(){ + community_operators_branch=$1 + all_checks_passed=false + if [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -iv pass | wc -l)" -eq 0 ] + then + export all_checks_passed=true + fi + echo "$all_checks_passed" +} + +passed_k8s_checks=$(did_all_checks_pass $community_operators_kubernetes_branch) +passed_openshift_checks=$(did_all_checks_pass $community_operators_openshift_branch) + +if [ $passed_k8s_checks == "false" ] || [ $passed_openshift_checks == "false" ] +then + echo "some test failed :(" + exit 1 +fi diff --git a/build/ci/check_demo_pr_checks.sh b/build/ci/check_demo_pr_checks.sh deleted file mode 100755 index a834abf7d..000000000 --- a/build/ci/check_demo_pr_checks.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -xe -set +o pipefail - -export passed_k8s_checks=true -export passed_openshift_checks=true - -wait_for_checks_to_complete(){ - community_operators_branch=$1 - is_summary_test_passed=false - export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` - if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then - sleep 5 - while [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do - sleep 15 - done - summary_counter=0 - export search_summary_test=`gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i summary` - while [ "$search_summary_test" != *"pending"* ] && [ $summary_counter -lt 20 ] && [ $is_summary_test_passed == "false" ]; do - sleep 1 - export search_summary_test=`gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i summary` - ((summary_counter=summary_counter+1)) - if [[ "$search_summary_test" == *"pass"* ]]; then - is_summary_test_passed=true - fi - done - if [[ $is_summary_test_passed == "false" ]]; then - while [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -i pending | wc -l)" -gt 0 ]; do - sleep 5 - done - fi - fi -} - -are_all_tests_were_passed(){ - community_operators_branch=$1 - all_checks_passed=false - if [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -iv pass | wc -l)" -eq 0 ] - then - export all_checks_passed=true - fi - echo "$all_checks_passed" -} - -wait_for_checks_to_complete $community_operators_kubernetes_branch -wait_for_checks_to_complete $community_operators_openshift_branch - -passed_k8s_checks=$(are_all_tests_were_passed $community_operators_kubernetes_branch) -passed_openshift_checks=$(are_all_tests_were_passed $community_operators_openshift_branch) - -if [ $passed_k8s_checks == "false" ] || [ $passed_openshift_checks == "false" ] -then - echo "some test failed :(" - exit 1 -fi diff --git a/build/ci/create_demo_pr.sh b/build/ci/create_demo_pr.sh index df328b637..444bd5fd8 100755 --- a/build/ci/create_demo_pr.sh +++ b/build/ci/create_demo_pr.sh @@ -1,7 +1,12 @@ #!/bin/bash -xe set +o pipefail -sed -i "s+$operator_image_in_branch+$operator_image_for_test+g" $csv_file +yq -y --arg operator_image $operator_image_for_test '.spec.install.spec.deployments[0].spec.template.spec.containers[0].image |= "$operator_image"' $csv_file > temp.yaml +cat temp.yaml > $csv_file +yq -y --arg operator_image $operator_image_for_test '.metadata.annotations.containerImage |= "$operator_image"' $csv_file > temp.yaml +cat temp.yaml > $csv_file +yq -y --arg operator_image $operator_image_for_test '..spec.relatedImages[0].image |= "$operator_image"' $csv_file > temp.yaml +cat temp.yaml > $csv_file echo $github_token > github_token.txt gh auth login --with-token < github_token.txt @@ -11,8 +16,6 @@ git remote set-url origin https://csiblock:$github_token@github.com/csiblock/com git fetch upstream git rebase upstream/master git push origin master --force -git config --global user.email csi.block1@il.ibm.com -git config --global user.name csiblock create_demo_pr(){ community_operators_branch=$1 diff --git a/build/ci/install_ci_dependencies.sh b/build/ci/setup_ci_dependencies.sh similarity index 76% rename from build/ci/install_ci_dependencies.sh rename to build/ci/setup_ci_dependencies.sh index abbae2b78..d6f443160 100755 --- a/build/ci/install_ci_dependencies.sh +++ b/build/ci/setup_ci_dependencies.sh @@ -2,9 +2,14 @@ set +o pipefail python -m pip install --upgrade pip yq +echo yq > dev-requirements.txt # install gh command curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null sudo apt-get update sudo apt-get install gh + +# configure git configuration +git config --global user.email csi.block1@il.ibm.com +git config --global user.name csiblock \ No newline at end of file diff --git a/build/ci/wait_demo_pr_checks.sh b/build/ci/wait_demo_pr_checks.sh new file mode 100755 index 000000000..59ba5f23a --- /dev/null +++ b/build/ci/wait_demo_pr_checks.sh @@ -0,0 +1,33 @@ +#!/bin/bash -xe +set +o pipefail + +wait_for_checks_to_complete(){ + community_operators_branch=$1 + all_tests_passed=false + gh_pr_checks_command='gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository' + export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` + if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then + sleep 5 + while [ "$(eval '$gh_pr_checks_command' | grep -i pending | wc -l)" -gt 0 ]; do + sleep 15 + done + summary_counter=0 + seconds_to_wait_for_summary_test=20 + while [ "$test_summary" != *"pending"* ] && [ $summary_counter -lt $seconds_to_wait_for_summary_test ] && [ $all_tests_passed == "false" ]; do + sleep 1 + export test_summary=`eval '$gh_pr_checks_command' | grep -i summary` + ((summary_counter=summary_counter+1)) + if [[ "$test_summary" == *"pass"* ]]; then + all_tests_passed=true + fi + done + if [[ $all_tests_passed == "false" ]]; then + while [ "$(eval '$gh_pr_checks_command' | grep -i pending | wc -l)" -gt 0 ]; do + sleep 5 + done + fi + fi +} + +wait_for_checks_to_complete $community_operators_kubernetes_branch +wait_for_checks_to_complete $community_operators_openshift_branch From bf9eac4222c56e4d90ebb0797bae77ee9bea376e Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Mon, 14 Jun 2021 22:13:28 +0300 Subject: [PATCH 23/51] fix image field in yq --- build/ci/create_demo_pr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci/create_demo_pr.sh b/build/ci/create_demo_pr.sh index 444bd5fd8..f29b61164 100755 --- a/build/ci/create_demo_pr.sh +++ b/build/ci/create_demo_pr.sh @@ -5,7 +5,7 @@ yq -y --arg operator_image $operator_image_for_test '.spec.install.spec.deployme cat temp.yaml > $csv_file yq -y --arg operator_image $operator_image_for_test '.metadata.annotations.containerImage |= "$operator_image"' $csv_file > temp.yaml cat temp.yaml > $csv_file -yq -y --arg operator_image $operator_image_for_test '..spec.relatedImages[0].image |= "$operator_image"' $csv_file > temp.yaml +yq -y --arg operator_image $operator_image_for_test '.spec.relatedImages[0].image |= "$operator_image"' $csv_file > temp.yaml cat temp.yaml > $csv_file echo $github_token > github_token.txt From afdc338240c1c19a7f7353eeb7ddb3fe8cced03f Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Mon, 14 Jun 2021 22:16:22 +0300 Subject: [PATCH 24/51] move variable to stage env --- .github/workflows/csi_operator_ci.yaml | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index e42c2437e..078ceadeb 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -22,10 +22,6 @@ jobs: name: "prepare env" needs: operator_image_build runs-on: ubuntu-latest - env: - github_csiblock_community_operators_repository: "csiblock/community-operators" - community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" - community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" steps: - name: Checkout code uses: actions/checkout@v2 @@ -65,6 +61,10 @@ jobs: needs: prepare_env runs-on: ubuntu-latest timeout-minutes: 30 + env: + github_csiblock_community_operators_repository: "csiblock/community-operators" + community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" + community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" steps: - name: Checkout code uses: actions/checkout@v2 @@ -85,9 +85,6 @@ jobs: run: | build/ci/create_demo_pr.sh env: - github_csiblock_community_operators_repository: ${{ env.github_csiblock_community_operators_repository }} - community_operators_kubernetes_branch: ${{ env.community_operators_kubernetes_branch }} - community_operators_openshift_branch: ${{ env.community_operators_openshift_branch }} github_token: ${{ secrets.CSIBLOCK_GITHUB_TOKEN }} repository_path: ${{ needs.prepare_env.outputs.repository_path }} operator_image_in_branch: ${{ needs.prepare_env.outputs.operator_image_in_branch }} @@ -96,22 +93,10 @@ jobs: - name: Wait for all the checks to finish running run: | build/ci/wait_demo_pr_checks.sh - env: - github_csiblock_community_operators_repository: ${{ env.github_csiblock_community_operators_repository }} - community_operators_kubernetes_branch: ${{ env.community_operators_kubernetes_branch }} - community_operators_openshift_branch: ${{ env.community_operators_openshift_branch }} - name: Assert all demo PR checks passed run: | build/ci/check_all_checks.sh - env: - github_csiblock_community_operators_repository: ${{ env.github_csiblock_community_operators_repository }} - community_operators_kubernetes_branch: ${{ env.community_operators_kubernetes_branch }} - community_operators_openshift_branch: ${{ env.community_operators_openshift_branch }} - name: Clean demo PR if: always() run: | build/ci/clean_demo_pr.sh - env: - github_csiblock_community_operators_repository: ${{ env.github_csiblock_community_operators_repository }} - community_operators_kubernetes_branch: ${{ env.community_operators_kubernetes_branch }} - community_operators_openshift_branch: ${{ env.community_operators_openshift_branch }} From 4584130e2bda806935ad28f5fe8b1f2ce3d31589 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Mon, 14 Jun 2021 22:39:43 +0300 Subject: [PATCH 25/51] ignore error on pr checks --- build/ci/clean_demo_pr.sh | 2 +- build/ci/create_demo_pr.sh | 6 +++--- build/ci/wait_demo_pr_checks.sh | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build/ci/clean_demo_pr.sh b/build/ci/clean_demo_pr.sh index 348085964..5c89fb49d 100755 --- a/build/ci/clean_demo_pr.sh +++ b/build/ci/clean_demo_pr.sh @@ -7,7 +7,7 @@ print_checks_and_delete_pr(){ export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then echo "The $cluster_kind checks:" - gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository + gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository || true gh pr close $community_operators_branch --delete-branch --repo $github_csiblock_community_operators_repository fi } diff --git a/build/ci/create_demo_pr.sh b/build/ci/create_demo_pr.sh index f29b61164..e509b3e9b 100755 --- a/build/ci/create_demo_pr.sh +++ b/build/ci/create_demo_pr.sh @@ -1,11 +1,11 @@ #!/bin/bash -xe set +o pipefail -yq -y --arg operator_image $operator_image_for_test '.spec.install.spec.deployments[0].spec.template.spec.containers[0].image |= "$operator_image"' $csv_file > temp.yaml +yq -y --arg operator_image $operator_image_for_test '.spec.install.spec.deployments[0].spec.template.spec.containers[0].image |= $operator_image' $csv_file > temp.yaml cat temp.yaml > $csv_file -yq -y --arg operator_image $operator_image_for_test '.metadata.annotations.containerImage |= "$operator_image"' $csv_file > temp.yaml +yq -y --arg operator_image $operator_image_for_test '.metadata.annotations.containerImage |= $operator_image' $csv_file > temp.yaml cat temp.yaml > $csv_file -yq -y --arg operator_image $operator_image_for_test '.spec.relatedImages[0].image |= "$operator_image"' $csv_file > temp.yaml +yq -y --arg operator_image $operator_image_for_test '.spec.relatedImages[0].image |= $operator_image' $csv_file > temp.yaml cat temp.yaml > $csv_file echo $github_token > github_token.txt diff --git a/build/ci/wait_demo_pr_checks.sh b/build/ci/wait_demo_pr_checks.sh index 59ba5f23a..7a13c2f1e 100755 --- a/build/ci/wait_demo_pr_checks.sh +++ b/build/ci/wait_demo_pr_checks.sh @@ -4,25 +4,25 @@ set +o pipefail wait_for_checks_to_complete(){ community_operators_branch=$1 all_tests_passed=false - gh_pr_checks_command='gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository' + gh_pr_checks_command=(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository) export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then sleep 5 - while [ "$(eval '$gh_pr_checks_command' | grep -i pending | wc -l)" -gt 0 ]; do + while [ `"${gh_pr_checks_command[@]}" | grep -i pending | wc -l` -eq 0 ]; do sleep 15 done summary_counter=0 seconds_to_wait_for_summary_test=20 while [ "$test_summary" != *"pending"* ] && [ $summary_counter -lt $seconds_to_wait_for_summary_test ] && [ $all_tests_passed == "false" ]; do sleep 1 - export test_summary=`eval '$gh_pr_checks_command' | grep -i summary` + export test_summary=`"${gh_pr_checks_command[@]}" | grep -i summary` ((summary_counter=summary_counter+1)) if [[ "$test_summary" == *"pass"* ]]; then all_tests_passed=true fi done if [[ $all_tests_passed == "false" ]]; then - while [ "$(eval '$gh_pr_checks_command' | grep -i pending | wc -l)" -gt 0 ]; do + while [ `"${gh_pr_checks_command[@]}" | grep -i pending | wc -l` -eq 0 ]; do sleep 5 done fi From 0672ae85cea2a7752cd1375355b95c6469abcf2a Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Mon, 14 Jun 2021 22:46:55 +0300 Subject: [PATCH 26/51] remove operator_image_in_branch variable --- .github/workflows/csi_operator_ci.yaml | 11 ++++------- build/ci/prepare_env.sh | 3 --- build/ci/wait_demo_pr_checks.sh | 4 ++-- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 078ceadeb..a64aaeafd 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -51,7 +51,6 @@ jobs: community_operators_path: "${{ steps.environment_setup.outputs.community_operators_path }}" upstream_community_operators_path: "${{ steps.environment_setup.outputs.upstream_community_operators_path }}" repository_path: "${{ steps.environment_setup.outputs.repository_path }}" - operator_image_in_branch: "${{ steps.environment_setup.outputs.operator_image_in_branch }}" operator_image_for_test: ${{ needs.operator_image_build.outputs.operator_image_for_test }} csv_file: ${{ steps.environment_setup.outputs.csv_file }} github_build_number: ${{ github.run_number }} @@ -65,6 +64,10 @@ jobs: github_csiblock_community_operators_repository: "csiblock/community-operators" community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" + github_token: ${{ secrets.CSIBLOCK_GITHUB_TOKEN }} + repository_path: ${{ needs.prepare_env.outputs.repository_path }} + operator_image_for_test: ${{ needs.prepare_env.outputs.operator_image_for_test }} + csv_file: ${{ needs.prepare_env.outputs.csv_file }} steps: - name: Checkout code uses: actions/checkout@v2 @@ -84,12 +87,6 @@ jobs: - name: Create demo PR run: | build/ci/create_demo_pr.sh - env: - github_token: ${{ secrets.CSIBLOCK_GITHUB_TOKEN }} - repository_path: ${{ needs.prepare_env.outputs.repository_path }} - operator_image_in_branch: ${{ needs.prepare_env.outputs.operator_image_in_branch }} - operator_image_for_test: ${{ needs.prepare_env.outputs.operator_image_for_test }} - csv_file: ${{ needs.prepare_env.outputs.csv_file }} - name: Wait for all the checks to finish running run: | build/ci/wait_demo_pr_checks.sh diff --git a/build/ci/prepare_env.sh b/build/ci/prepare_env.sh index c93a47d10..3d248529c 100755 --- a/build/ci/prepare_env.sh +++ b/build/ci/prepare_env.sh @@ -6,12 +6,9 @@ export csv_version=`echo ${csv_version//ibm-block-csi-operator.v}` export csv_version=`echo ${csv_version//\"}` export repository_path=~/work/$CI_REPOSITORY_NAME/$CI_REPOSITORY_NAME export csv_file=$repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$csv_version/ibm-block-csi-operator.v$csv_version.clusterserviceversion.yaml -export operator_image_in_branch=`yq .metadata.annotations.containerImage $csv_file` -export operator_image_in_branch=`echo ${operator_image_in_branch//\"}` export upstream_community_operators_path=upstream-community-operators/ibm-block-csi-operator-community/$csv_version export community_operators_path=community-operators/ibm-block-csi-operator-community/$csv_version -echo "::set-output name=operator_image_in_branch::${operator_image_in_branch}" echo "::set-output name=upstream_community_operators_path::${upstream_community_operators_path}" echo "::set-output name=community_operators_path::${community_operators_path}" echo "::set-output name=repository_path::${repository_path}" diff --git a/build/ci/wait_demo_pr_checks.sh b/build/ci/wait_demo_pr_checks.sh index 7a13c2f1e..5b0bd905d 100755 --- a/build/ci/wait_demo_pr_checks.sh +++ b/build/ci/wait_demo_pr_checks.sh @@ -8,7 +8,7 @@ wait_for_checks_to_complete(){ export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then sleep 5 - while [ `"${gh_pr_checks_command[@]}" | grep -i pending | wc -l` -eq 0 ]; do + while [ `"${gh_pr_checks_command[@]}" | grep -i pending | wc -l` -gt 0 ]; do sleep 15 done summary_counter=0 @@ -22,7 +22,7 @@ wait_for_checks_to_complete(){ fi done if [[ $all_tests_passed == "false" ]]; then - while [ `"${gh_pr_checks_command[@]}" | grep -i pending | wc -l` -eq 0 ]; do + while [ `"${gh_pr_checks_command[@]}" | grep -i pending | wc -l` -gt 0 ]; do sleep 5 done fi From 7e9d3053b725f489897aafdc7335a43d6e352494 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Tue, 15 Jun 2021 09:20:57 +0300 Subject: [PATCH 27/51] add github_build_number env parameter --- .github/workflows/csi_operator_ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index a64aaeafd..0fb4f33c9 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -53,7 +53,6 @@ jobs: repository_path: "${{ steps.environment_setup.outputs.repository_path }}" operator_image_for_test: ${{ needs.operator_image_build.outputs.operator_image_for_test }} csv_file: ${{ steps.environment_setup.outputs.csv_file }} - github_build_number: ${{ github.run_number }} CI-csi-demo-pr: name: "community-operators fork checks" @@ -68,6 +67,7 @@ jobs: repository_path: ${{ needs.prepare_env.outputs.repository_path }} operator_image_for_test: ${{ needs.prepare_env.outputs.operator_image_for_test }} csv_file: ${{ needs.prepare_env.outputs.csv_file }} + github_build_number: ${{ github.run_number }} steps: - name: Checkout code uses: actions/checkout@v2 From 88bd0a3450ea562f5fb347276f580afcc683d5bd Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 16 Jun 2021 09:49:04 +0300 Subject: [PATCH 28/51] moved scripts to community dir --- .github/workflows/csi_operator_ci.yaml | 14 +++++++------- build/ci/{ => community}/check_all_checks.sh | 0 build/ci/{ => community}/clean_demo_pr.sh | 0 build/ci/{ => community}/create_demo_pr.sh | 0 build/ci/{ => community}/prepare_env.sh | 4 ++-- build/ci/{ => community}/setup_ci_dependencies.sh | 4 +--- build/ci/{ => community}/wait_demo_pr_checks.sh | 14 +++++++------- 7 files changed, 17 insertions(+), 19 deletions(-) rename build/ci/{ => community}/check_all_checks.sh (100%) rename build/ci/{ => community}/clean_demo_pr.sh (100%) rename build/ci/{ => community}/create_demo_pr.sh (100%) rename build/ci/{ => community}/prepare_env.sh (79%) rename build/ci/{ => community}/setup_ci_dependencies.sh (86%) rename build/ci/{ => community}/wait_demo_pr_checks.sh (53%) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 0fb4f33c9..7988b1094 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -31,7 +31,7 @@ jobs: python-version: '3.x' - name: Setup dependencies run: | - build/ci/setup_ci_dependencies.sh + build/ci/community/setup_ci_dependencies.sh - uses: actions/cache@v2 with: path: ${{ env.pythonLocation }} @@ -46,7 +46,7 @@ jobs: - name: Prepare env id: environment_setup run: | - build/ci/prepare_env.sh + build/ci/community/prepare_env.sh outputs: community_operators_path: "${{ steps.environment_setup.outputs.community_operators_path }}" upstream_community_operators_path: "${{ steps.environment_setup.outputs.upstream_community_operators_path }}" @@ -86,14 +86,14 @@ jobs: path: /home/runner - name: Create demo PR run: | - build/ci/create_demo_pr.sh - - name: Wait for all the checks to finish running + build/ci/community/create_demo_pr.sh + - name: Wait for all demo PR checks to finish run: | - build/ci/wait_demo_pr_checks.sh + build/ci/community/wait_demo_pr_checks.sh - name: Assert all demo PR checks passed run: | - build/ci/check_all_checks.sh + build/ci/community/check_all_checks.sh - name: Clean demo PR if: always() run: | - build/ci/clean_demo_pr.sh + build/ci/community/clean_demo_pr.sh diff --git a/build/ci/check_all_checks.sh b/build/ci/community/check_all_checks.sh similarity index 100% rename from build/ci/check_all_checks.sh rename to build/ci/community/check_all_checks.sh diff --git a/build/ci/clean_demo_pr.sh b/build/ci/community/clean_demo_pr.sh similarity index 100% rename from build/ci/clean_demo_pr.sh rename to build/ci/community/clean_demo_pr.sh diff --git a/build/ci/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh similarity index 100% rename from build/ci/create_demo_pr.sh rename to build/ci/community/create_demo_pr.sh diff --git a/build/ci/prepare_env.sh b/build/ci/community/prepare_env.sh similarity index 79% rename from build/ci/prepare_env.sh rename to build/ci/community/prepare_env.sh index 3d248529c..f14ec4f74 100755 --- a/build/ci/prepare_env.sh +++ b/build/ci/community/prepare_env.sh @@ -1,8 +1,8 @@ #!/bin/bash -xe set +o pipefail -export csv_version=`yq .channels[0].currentCSV deploy/olm-catalog/ibm-block-csi-operator-community/ibm-block-csi-operator.package.yaml` -export csv_version=`echo ${csv_version//ibm-block-csi-operator.v}` +csv_version=`yq .channels[0].currentCSV deploy/olm-catalog/ibm-block-csi-operator-community/ibm-block-csi-operator.package.yaml` +csv_version=`echo ${csv_version//ibm-block-csi-operator.v}` export csv_version=`echo ${csv_version//\"}` export repository_path=~/work/$CI_REPOSITORY_NAME/$CI_REPOSITORY_NAME export csv_file=$repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$csv_version/ibm-block-csi-operator.v$csv_version.clusterserviceversion.yaml diff --git a/build/ci/setup_ci_dependencies.sh b/build/ci/community/setup_ci_dependencies.sh similarity index 86% rename from build/ci/setup_ci_dependencies.sh rename to build/ci/community/setup_ci_dependencies.sh index d6f443160..33ae01525 100755 --- a/build/ci/setup_ci_dependencies.sh +++ b/build/ci/community/setup_ci_dependencies.sh @@ -4,12 +4,10 @@ set +o pipefail python -m pip install --upgrade pip yq echo yq > dev-requirements.txt -# install gh command curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null sudo apt-get update sudo apt-get install gh -# configure git configuration git config --global user.email csi.block1@il.ibm.com -git config --global user.name csiblock \ No newline at end of file +git config --global user.name csiblock diff --git a/build/ci/wait_demo_pr_checks.sh b/build/ci/community/wait_demo_pr_checks.sh similarity index 53% rename from build/ci/wait_demo_pr_checks.sh rename to build/ci/community/wait_demo_pr_checks.sh index 5b0bd905d..ebba0bbcd 100755 --- a/build/ci/wait_demo_pr_checks.sh +++ b/build/ci/community/wait_demo_pr_checks.sh @@ -4,25 +4,25 @@ set +o pipefail wait_for_checks_to_complete(){ community_operators_branch=$1 all_tests_passed=false - gh_pr_checks_command=(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository) + gh_pr_checks_command="`gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository`" export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then sleep 5 - while [ `"${gh_pr_checks_command[@]}" | grep -i pending | wc -l` -gt 0 ]; do + while [ `eval gh_pr_checks_command | grep -i pending | wc -l` -gt 0 ]; do sleep 15 done - summary_counter=0 + seconds_waited_for_summary_test=0 seconds_to_wait_for_summary_test=20 - while [ "$test_summary" != *"pending"* ] && [ $summary_counter -lt $seconds_to_wait_for_summary_test ] && [ $all_tests_passed == "false" ]; do + while [ "$test_summary" != *"pending"* ] && [ $seconds_waited_for_summary_test -lt $summary_test_timeout_seconds] && [ $all_tests_passed == "false" ]; do sleep 1 - export test_summary=`"${gh_pr_checks_command[@]}" | grep -i summary` - ((summary_counter=summary_counter+1)) + test_summary=`eval gh_pr_checks_command | grep -i summary` + ((seconds_waited_for_summary_test=seconds_waited_for_summary_test+1)) if [[ "$test_summary" == *"pass"* ]]; then all_tests_passed=true fi done if [[ $all_tests_passed == "false" ]]; then - while [ `"${gh_pr_checks_command[@]}" | grep -i pending | wc -l` -gt 0 ]; do + while [ `eval gh_pr_checks_command | grep -i pending | wc -l` -gt 0 ]; do sleep 5 done fi From 9386785b525fa76d5611e54dc62c06bc6517f0c7 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 16 Jun 2021 10:09:42 +0300 Subject: [PATCH 29/51] remove wrong ` --- build/ci/community/wait_demo_pr_checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci/community/wait_demo_pr_checks.sh b/build/ci/community/wait_demo_pr_checks.sh index ebba0bbcd..c6d9bf0c0 100755 --- a/build/ci/community/wait_demo_pr_checks.sh +++ b/build/ci/community/wait_demo_pr_checks.sh @@ -4,7 +4,7 @@ set +o pipefail wait_for_checks_to_complete(){ community_operators_branch=$1 all_tests_passed=false - gh_pr_checks_command="`gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository`" + gh_pr_checks_command="gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository" export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then sleep 5 From 5eff30ae6f4850e69a5ffa9d2145dd6bd9674df7 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 16 Jun 2021 10:14:44 +0300 Subject: [PATCH 30/51] call gh_pr_checks_command as variable --- build/ci/community/wait_demo_pr_checks.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/ci/community/wait_demo_pr_checks.sh b/build/ci/community/wait_demo_pr_checks.sh index c6d9bf0c0..5e66d7290 100755 --- a/build/ci/community/wait_demo_pr_checks.sh +++ b/build/ci/community/wait_demo_pr_checks.sh @@ -8,21 +8,21 @@ wait_for_checks_to_complete(){ export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then sleep 5 - while [ `eval gh_pr_checks_command | grep -i pending | wc -l` -gt 0 ]; do + while [ `eval $gh_pr_checks_command | grep -i pending | wc -l` -gt 0 ]; do sleep 15 done seconds_waited_for_summary_test=0 seconds_to_wait_for_summary_test=20 while [ "$test_summary" != *"pending"* ] && [ $seconds_waited_for_summary_test -lt $summary_test_timeout_seconds] && [ $all_tests_passed == "false" ]; do sleep 1 - test_summary=`eval gh_pr_checks_command | grep -i summary` + test_summary=`eval $gh_pr_checks_command | grep -i summary` ((seconds_waited_for_summary_test=seconds_waited_for_summary_test+1)) if [[ "$test_summary" == *"pass"* ]]; then all_tests_passed=true fi done if [[ $all_tests_passed == "false" ]]; then - while [ `eval gh_pr_checks_command | grep -i pending | wc -l` -gt 0 ]; do + while [ `eval $gh_pr_checks_command | grep -i pending | wc -l` -gt 0 ]; do sleep 5 done fi From c0140d0925625849af86d21b2758cf3032301968 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Sun, 20 Jun 2021 16:00:06 +0300 Subject: [PATCH 31/51] changed yq stracture --- .github/workflows/csi_operator_ci.yaml | 18 ++++++---------- build/ci/community/check_all_checks.sh | 8 +++---- build/ci/community/clean_demo_pr.sh | 6 +++--- build/ci/community/create_demo_pr.sh | 15 ++++++------- build/ci/community/prepare_env.sh | 5 ++--- build/ci/community/setup_ci_dependencies.sh | 3 +-- build/ci/community/wait_demo_pr_checks.sh | 24 ++++++--------------- 7 files changed, 29 insertions(+), 50 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 7988b1094..46e2c56cf 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -25,26 +25,21 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - name: Setup dependencies run: | build/ci/community/setup_ci_dependencies.sh - - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('dev-requirements.txt') }} - - name: Save gitconfig file + - name: Save dependencies files uses: actions/upload-artifact@v2 with: name: gitconfig-file - path: /home/runner/.gitconfig + path: | + /home/runner/.gitconfig + /home/runner/.bash_profile retention-days: 1 - uses: FranzDiebold/github-env-vars-action@v2 - name: Prepare env id: environment_setup + shell: bash -l {0} run: | build/ci/community/prepare_env.sh outputs: @@ -60,7 +55,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 env: - github_csiblock_community_operators_repository: "csiblock/community-operators" + forked_community_operators_repository: "csiblock/community-operators" community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" github_token: ${{ secrets.CSIBLOCK_GITHUB_TOKEN }} @@ -85,6 +80,7 @@ jobs: name: gitconfig-file path: /home/runner - name: Create demo PR + shell: bash -l {0} run: | build/ci/community/create_demo_pr.sh - name: Wait for all demo PR checks to finish diff --git a/build/ci/community/check_all_checks.sh b/build/ci/community/check_all_checks.sh index 8f21f556f..49f6bb7b6 100755 --- a/build/ci/community/check_all_checks.sh +++ b/build/ci/community/check_all_checks.sh @@ -1,15 +1,15 @@ #!/bin/bash -xe set +o pipefail -export passed_k8s_checks=true -export passed_openshift_checks=true +export passed_k8s_checks=false +export passed_openshift_checks=false did_all_checks_pass(){ community_operators_branch=$1 all_checks_passed=false - if [ "$(gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository | grep -iv pass | wc -l)" -eq 0 ] + if [ "$(gh pr checks $community_operators_branch --repo $forked_community_operators_repository | grep -iv pass | wc -l)" -eq 0 ] then - export all_checks_passed=true + all_checks_passed=true fi echo "$all_checks_passed" } diff --git a/build/ci/community/clean_demo_pr.sh b/build/ci/community/clean_demo_pr.sh index 5c89fb49d..04f212137 100755 --- a/build/ci/community/clean_demo_pr.sh +++ b/build/ci/community/clean_demo_pr.sh @@ -4,11 +4,11 @@ set +o pipefail print_checks_and_delete_pr(){ community_operators_branch=$1 cluster_kind=$2 - export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` + export repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then echo "The $cluster_kind checks:" - gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository || true - gh pr close $community_operators_branch --delete-branch --repo $github_csiblock_community_operators_repository + gh pr checks $community_operators_branch --repo $forked_community_operators_repository || true + gh pr close $community_operators_branch --delete-branch --repo $forked_community_operators_repository fi } diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index e509b3e9b..325eec655 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -1,12 +1,9 @@ #!/bin/bash -xe set +o pipefail -yq -y --arg operator_image $operator_image_for_test '.spec.install.spec.deployments[0].spec.template.spec.containers[0].image |= $operator_image' $csv_file > temp.yaml -cat temp.yaml > $csv_file -yq -y --arg operator_image $operator_image_for_test '.metadata.annotations.containerImage |= $operator_image' $csv_file > temp.yaml -cat temp.yaml > $csv_file -yq -y --arg operator_image $operator_image_for_test '.spec.relatedImages[0].image |= $operator_image' $csv_file > temp.yaml -cat temp.yaml > $csv_file +yq eval ".spec.install.spec.deployments[0].spec.template.spec.containers[0].image |= env(operator_image_for_test)" $csv_file -i +yq eval ".metadata.annotations.containerImage |= env(operator_image_for_test)" $csv_file -i +yq eval ".spec.relatedImages[0].image |= env(operator_image_for_test)" $csv_file -i echo $github_token > github_token.txt gh auth login --with-token < github_token.txt @@ -21,9 +18,9 @@ create_demo_pr(){ community_operators_branch=$1 dest_path=$2 cluster_kind=$3 - export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` + export repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then - gh pr close $community_operators_branch --delete-branch --repo $github_csiblock_community_operators_repository + gh pr close $community_operators_branch --delete-branch --repo $forked_community_operators_repository fi git checkout master git checkout -b $community_operators_branch @@ -31,7 +28,7 @@ create_demo_pr(){ git add . git commit --signoff -m "build number $github_build_number $cluster_kind" git push origin $community_operators_branch - gh pr create --title "IBM Block CSI update $cluster_kind" --repo $github_csiblock_community_operators_repository --base master --head $community_operators_branch --body "pr check" + gh pr create --title "IBM Block CSI update $cluster_kind" --repo $forked_community_operators_repository --base master --head $community_operators_branch --body "pr check" } create_demo_pr $community_operators_kubernetes_branch "upstream-community-operators/" "kubernetes" diff --git a/build/ci/community/prepare_env.sh b/build/ci/community/prepare_env.sh index f14ec4f74..f5e7b4ae0 100755 --- a/build/ci/community/prepare_env.sh +++ b/build/ci/community/prepare_env.sh @@ -1,9 +1,8 @@ #!/bin/bash -xe set +o pipefail -csv_version=`yq .channels[0].currentCSV deploy/olm-catalog/ibm-block-csi-operator-community/ibm-block-csi-operator.package.yaml` -csv_version=`echo ${csv_version//ibm-block-csi-operator.v}` -export csv_version=`echo ${csv_version//\"}` +csv_version=`yq eval .channels[0].currentCSV deploy/olm-catalog/ibm-block-csi-operator-community/ibm-block-csi-operator.package.yaml` +export csv_version=`echo ${csv_version//ibm-block-csi-operator.v}` export repository_path=~/work/$CI_REPOSITORY_NAME/$CI_REPOSITORY_NAME export csv_file=$repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$csv_version/ibm-block-csi-operator.v$csv_version.clusterserviceversion.yaml export upstream_community_operators_path=upstream-community-operators/ibm-block-csi-operator-community/$csv_version diff --git a/build/ci/community/setup_ci_dependencies.sh b/build/ci/community/setup_ci_dependencies.sh index 33ae01525..d4dde766f 100755 --- a/build/ci/community/setup_ci_dependencies.sh +++ b/build/ci/community/setup_ci_dependencies.sh @@ -1,8 +1,7 @@ #!/bin/bash -xe set +o pipefail -python -m pip install --upgrade pip yq -echo yq > dev-requirements.txt +echo $'yq() {\n docker run --rm -e operator_image_for_test=$operator_image_for_test -i -v "${PWD}":/workdir mikefarah/yq "$@"\n}' >> /home/runner/.bash_profile curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null diff --git a/build/ci/community/wait_demo_pr_checks.sh b/build/ci/community/wait_demo_pr_checks.sh index 5e66d7290..0a20172fc 100755 --- a/build/ci/community/wait_demo_pr_checks.sh +++ b/build/ci/community/wait_demo_pr_checks.sh @@ -4,28 +4,16 @@ set +o pipefail wait_for_checks_to_complete(){ community_operators_branch=$1 all_tests_passed=false - gh_pr_checks_command="gh pr checks $community_operators_branch --repo $github_csiblock_community_operators_repository" - export repo_pr=`gh pr list --repo $github_csiblock_community_operators_repository | grep $community_operators_branch` + gh_pr_checks_command="gh pr checks $community_operators_branch --repo $forked_community_operators_repository" + repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then - sleep 5 - while [ `eval $gh_pr_checks_command | grep -i pending | wc -l` -gt 0 ]; do - sleep 15 + while [ `$gh_pr_checks_command | grep -i pending | wc -l` -eq 0 ]; do + sleep 1 done - seconds_waited_for_summary_test=0 - seconds_to_wait_for_summary_test=20 - while [ "$test_summary" != *"pending"* ] && [ $seconds_waited_for_summary_test -lt $summary_test_timeout_seconds] && [ $all_tests_passed == "false" ]; do + test_summary=`$gh_pr_checks_command | grep -i summary` + while [ "$test_summary" == *"pass"* ]; do sleep 1 - test_summary=`eval $gh_pr_checks_command | grep -i summary` - ((seconds_waited_for_summary_test=seconds_waited_for_summary_test+1)) - if [[ "$test_summary" == *"pass"* ]]; then - all_tests_passed=true - fi done - if [[ $all_tests_passed == "false" ]]; then - while [ `eval $gh_pr_checks_command | grep -i pending | wc -l` -gt 0 ]; do - sleep 5 - done - fi fi } From 79bf3dc209d54166a4739a9188e83d2c2f4363e3 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Mon, 21 Jun 2021 11:55:36 +0300 Subject: [PATCH 32/51] removed shell from github actions --- .github/workflows/csi_operator_ci.yaml | 10 ---------- build/ci/community/create_demo_pr.sh | 10 ++++++---- build/ci/community/prepare_env.sh | 2 +- build/ci/community/wait_demo_pr_checks.sh | 14 +++++++++----- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 46e2c56cf..0c92bae53 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -39,7 +39,6 @@ jobs: - uses: FranzDiebold/github-env-vars-action@v2 - name: Prepare env id: environment_setup - shell: bash -l {0} run: | build/ci/community/prepare_env.sh outputs: @@ -66,21 +65,12 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - uses: actions/cache@v2 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-${{ hashFiles('dev-requirements.txt') }} - name: Retrieve gitconfig file uses: actions/download-artifact@v2 with: name: gitconfig-file path: /home/runner - name: Create demo PR - shell: bash -l {0} run: | build/ci/community/create_demo_pr.sh - name: Wait for all demo PR checks to finish diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index 325eec655..36e648854 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -1,9 +1,11 @@ -#!/bin/bash -xe +#!/bin/bash -xel set +o pipefail -yq eval ".spec.install.spec.deployments[0].spec.template.spec.containers[0].image |= env(operator_image_for_test)" $csv_file -i -yq eval ".metadata.annotations.containerImage |= env(operator_image_for_test)" $csv_file -i -yq eval ".spec.relatedImages[0].image |= env(operator_image_for_test)" $csv_file -i +cd $(dirname $csv_file) +yq eval ".spec.install.spec.deployments[0].spec.template.spec.containers[0].image |= env(operator_image_for_test)" $(basename $csv_file) -i +yq eval ".metadata.annotations.containerImage |= env(operator_image_for_test)" $(basename $csv_file) -i +yq eval ".spec.relatedImages[0].image |= env(operator_image_for_test)" $(basename $csv_file) -i +cd - echo $github_token > github_token.txt gh auth login --with-token < github_token.txt diff --git a/build/ci/community/prepare_env.sh b/build/ci/community/prepare_env.sh index f5e7b4ae0..c4228caff 100755 --- a/build/ci/community/prepare_env.sh +++ b/build/ci/community/prepare_env.sh @@ -1,4 +1,4 @@ -#!/bin/bash -xe +#!/bin/bash -xel set +o pipefail csv_version=`yq eval .channels[0].currentCSV deploy/olm-catalog/ibm-block-csi-operator-community/ibm-block-csi-operator.package.yaml` diff --git a/build/ci/community/wait_demo_pr_checks.sh b/build/ci/community/wait_demo_pr_checks.sh index 0a20172fc..2b0128219 100755 --- a/build/ci/community/wait_demo_pr_checks.sh +++ b/build/ci/community/wait_demo_pr_checks.sh @@ -1,17 +1,21 @@ #!/bin/bash -xe set +o pipefail +wait_fot_checks_to_start(){ + gh_pr_checks_command=$1 + while [ `$gh_pr_checks_command | grep -i pending | wc -l` -eq 0 ]; do + sleep 1 + done +} wait_for_checks_to_complete(){ community_operators_branch=$1 all_tests_passed=false gh_pr_checks_command="gh pr checks $community_operators_branch --repo $forked_community_operators_repository" repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then - while [ `$gh_pr_checks_command | grep -i pending | wc -l` -eq 0 ]; do - sleep 1 - done - test_summary=`$gh_pr_checks_command | grep -i summary` - while [ "$test_summary" == *"pass"* ]; do + wait_fot_checks_to_start $gh_pr_checks_command + test_summary="$gh_pr_checks_command | grep -i summary" + while [[ ! "$test_summary" =~ "pass" ]] && [[ ! "$test_summary" =~ "fail" ]]; do sleep 1 done fi From c30c38162b4c21885891118cb8b51c4f4b546691 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Mon, 21 Jun 2021 12:03:38 +0300 Subject: [PATCH 33/51] add parameters to wait_fot_checks_to_start --- build/ci/community/wait_demo_pr_checks.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build/ci/community/wait_demo_pr_checks.sh b/build/ci/community/wait_demo_pr_checks.sh index 2b0128219..ffd2496f2 100755 --- a/build/ci/community/wait_demo_pr_checks.sh +++ b/build/ci/community/wait_demo_pr_checks.sh @@ -2,7 +2,9 @@ set +o pipefail wait_fot_checks_to_start(){ - gh_pr_checks_command=$1 + community_operators_branch=$1 + forked_community_operators_repository=$2 + gh_pr_checks_command="gh pr checks $community_operators_branch --repo $forked_community_operators_repository" while [ `$gh_pr_checks_command | grep -i pending | wc -l` -eq 0 ]; do sleep 1 done @@ -13,7 +15,7 @@ wait_for_checks_to_complete(){ gh_pr_checks_command="gh pr checks $community_operators_branch --repo $forked_community_operators_repository" repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then - wait_fot_checks_to_start $gh_pr_checks_command + wait_fot_checks_to_start $community_operators_branch $forked_community_operators_repository test_summary="$gh_pr_checks_command | grep -i summary" while [[ ! "$test_summary" =~ "pass" ]] && [[ ! "$test_summary" =~ "fail" ]]; do sleep 1 From 219b1a256a326fe99af7c96ef024bbd5bbffc271 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 30 Jun 2021 15:20:04 +0300 Subject: [PATCH 34/51] change while check --- build/ci/community/wait_demo_pr_checks.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/ci/community/wait_demo_pr_checks.sh b/build/ci/community/wait_demo_pr_checks.sh index ffd2496f2..e269132cc 100755 --- a/build/ci/community/wait_demo_pr_checks.sh +++ b/build/ci/community/wait_demo_pr_checks.sh @@ -16,8 +16,7 @@ wait_for_checks_to_complete(){ repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then wait_fot_checks_to_start $community_operators_branch $forked_community_operators_repository - test_summary="$gh_pr_checks_command | grep -i summary" - while [[ ! "$test_summary" =~ "pass" ]] && [[ ! "$test_summary" =~ "fail" ]]; do + while [[ ! "`$gh_pr_checks_command | grep -i summary`" =~ "pass" ]] && [[ ! "`$gh_pr_checks_command | grep -i summary`" =~ "fail" ]]; do sleep 1 done fi From 28777cf60d2eec91f4fae195e3ffa016ac7bd283 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 30 Jun 2021 15:30:04 +0300 Subject: [PATCH 35/51] remove exports --- build/ci/community/check_all_checks.sh | 3 --- build/ci/community/clean_demo_pr.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/build/ci/community/check_all_checks.sh b/build/ci/community/check_all_checks.sh index 49f6bb7b6..429975a6d 100755 --- a/build/ci/community/check_all_checks.sh +++ b/build/ci/community/check_all_checks.sh @@ -1,9 +1,6 @@ #!/bin/bash -xe set +o pipefail -export passed_k8s_checks=false -export passed_openshift_checks=false - did_all_checks_pass(){ community_operators_branch=$1 all_checks_passed=false diff --git a/build/ci/community/clean_demo_pr.sh b/build/ci/community/clean_demo_pr.sh index 04f212137..99029d549 100755 --- a/build/ci/community/clean_demo_pr.sh +++ b/build/ci/community/clean_demo_pr.sh @@ -4,7 +4,7 @@ set +o pipefail print_checks_and_delete_pr(){ community_operators_branch=$1 cluster_kind=$2 - export repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` + repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then echo "The $cluster_kind checks:" gh pr checks $community_operators_branch --repo $forked_community_operators_repository || true From e52e5f95eba5ffc7666e47d9d6c5ea3dcdc0871b Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 30 Jun 2021 17:00:08 +0300 Subject: [PATCH 36/51] add loop for editing the csv file --- build/ci/community/create_demo_pr.sh | 12 +++++++++--- build/ci/community/prepare_env.sh | 4 ++-- build/ci/community/setup_ci_dependencies.sh | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index 36e648854..a8015b423 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -2,9 +2,15 @@ set +o pipefail cd $(dirname $csv_file) -yq eval ".spec.install.spec.deployments[0].spec.template.spec.containers[0].image |= env(operator_image_for_test)" $(basename $csv_file) -i -yq eval ".metadata.annotations.containerImage |= env(operator_image_for_test)" $(basename $csv_file) -i -yq eval ".spec.relatedImages[0].image |= env(operator_image_for_test)" $(basename $csv_file) -i +declare -a operator_image_fields=( + ".spec.install.spec.deployments[0].spec.template.spec.containers[0].image" + ".metadata.annotations.containerImage" + ".spec.relatedImages[0].image" +) +for image_field in "${operator_image_fields[@]}" +do + yq eval "$image_field |= env(operator_image_for_test)" $(basename $csv_file) -i +done cd - echo $github_token > github_token.txt diff --git a/build/ci/community/prepare_env.sh b/build/ci/community/prepare_env.sh index c4228caff..03c64ae06 100755 --- a/build/ci/community/prepare_env.sh +++ b/build/ci/community/prepare_env.sh @@ -1,7 +1,7 @@ #!/bin/bash -xel set +o pipefail - -csv_version=`yq eval .channels[0].currentCSV deploy/olm-catalog/ibm-block-csi-operator-community/ibm-block-csi-operator.package.yaml` +operator_package_file_path=deploy/olm-catalog/ibm-block-csi-operator-community/ibm-block-csi-operator.package.yaml +csv_version=`yq eval .channels[0].currentCSV $operator_package_file_path` export csv_version=`echo ${csv_version//ibm-block-csi-operator.v}` export repository_path=~/work/$CI_REPOSITORY_NAME/$CI_REPOSITORY_NAME export csv_file=$repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$csv_version/ibm-block-csi-operator.v$csv_version.clusterserviceversion.yaml diff --git a/build/ci/community/setup_ci_dependencies.sh b/build/ci/community/setup_ci_dependencies.sh index d4dde766f..28ea792fa 100755 --- a/build/ci/community/setup_ci_dependencies.sh +++ b/build/ci/community/setup_ci_dependencies.sh @@ -1,7 +1,7 @@ #!/bin/bash -xe set +o pipefail -echo $'yq() {\n docker run --rm -e operator_image_for_test=$operator_image_for_test -i -v "${PWD}":/workdir mikefarah/yq "$@"\n}' >> /home/runner/.bash_profile +echo $'yq() {\n docker run --rm -e operator_image_for_test=$operator_image_for_test -i -v "${PWD}":/workdir mikefarah/yq:4 "$@"\n}' >> /home/runner/.bash_profile curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null From 8bf24e140d88eee1962d160780ce8ab109a7701d Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 1 Jul 2021 09:41:55 +0300 Subject: [PATCH 37/51] remove exports --- build/ci/community/create_demo_pr.sh | 2 +- build/ci/community/wait_demo_pr_checks.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index a8015b423..b2fa4c908 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -26,7 +26,7 @@ create_demo_pr(){ community_operators_branch=$1 dest_path=$2 cluster_kind=$3 - export repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` + repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then gh pr close $community_operators_branch --delete-branch --repo $forked_community_operators_repository fi diff --git a/build/ci/community/wait_demo_pr_checks.sh b/build/ci/community/wait_demo_pr_checks.sh index e269132cc..6c87d6461 100755 --- a/build/ci/community/wait_demo_pr_checks.sh +++ b/build/ci/community/wait_demo_pr_checks.sh @@ -16,7 +16,8 @@ wait_for_checks_to_complete(){ repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then wait_fot_checks_to_start $community_operators_branch $forked_community_operators_repository - while [[ ! "`$gh_pr_checks_command | grep -i summary`" =~ "pass" ]] && [[ ! "`$gh_pr_checks_command | grep -i summary`" =~ "fail" ]]; do + test_summary="$gh_pr_checks_command | grep -i summary" + while [[ ! "`eval $test_summary`" =~ "pass" ]] && [[ ! "`eval $test_summary`" =~ "fail" ]]; do sleep 1 done fi From 2f2a1bd226521281da68527f9a3a8a3190517020 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 1 Jul 2021 09:45:25 +0300 Subject: [PATCH 38/51] eval repo_pr --- build/ci/community/clean_demo_pr.sh | 2 +- build/ci/community/create_demo_pr.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/ci/community/clean_demo_pr.sh b/build/ci/community/clean_demo_pr.sh index 99029d549..68c1a3a9b 100755 --- a/build/ci/community/clean_demo_pr.sh +++ b/build/ci/community/clean_demo_pr.sh @@ -5,7 +5,7 @@ print_checks_and_delete_pr(){ community_operators_branch=$1 cluster_kind=$2 repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` - if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then + if [[ "`eval $repo_pr`" == *"$community_operators_branch"* ]]; then echo "The $cluster_kind checks:" gh pr checks $community_operators_branch --repo $forked_community_operators_repository || true gh pr close $community_operators_branch --delete-branch --repo $forked_community_operators_repository diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index b2fa4c908..f560d3074 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -27,7 +27,7 @@ create_demo_pr(){ dest_path=$2 cluster_kind=$3 repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` - if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then + if [[ "`eval $repo_pr`" == *"$community_operators_branch"* ]]; then gh pr close $community_operators_branch --delete-branch --repo $forked_community_operators_repository fi git checkout master From 0b6a6b566c46aeac5188a8c2ce3bc68c20c59cad Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Thu, 1 Jul 2021 09:57:42 +0300 Subject: [PATCH 39/51] change repo_pr --- build/ci/community/clean_demo_pr.sh | 2 +- build/ci/community/create_demo_pr.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/ci/community/clean_demo_pr.sh b/build/ci/community/clean_demo_pr.sh index 68c1a3a9b..f6209cc5b 100755 --- a/build/ci/community/clean_demo_pr.sh +++ b/build/ci/community/clean_demo_pr.sh @@ -4,7 +4,7 @@ set +o pipefail print_checks_and_delete_pr(){ community_operators_branch=$1 cluster_kind=$2 - repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` + repo_pr="gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch" if [[ "`eval $repo_pr`" == *"$community_operators_branch"* ]]; then echo "The $cluster_kind checks:" gh pr checks $community_operators_branch --repo $forked_community_operators_repository || true diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index f560d3074..610193ea7 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -26,7 +26,7 @@ create_demo_pr(){ community_operators_branch=$1 dest_path=$2 cluster_kind=$3 - repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` + repo_pr="gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch" if [[ "`eval $repo_pr`" == *"$community_operators_branch"* ]]; then gh pr close $community_operators_branch --delete-branch --repo $forked_community_operators_repository fi From abff90b1048472867c59dfd4dca248d6718a4023 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Mon, 12 Jul 2021 14:27:26 +0300 Subject: [PATCH 40/51] add some tickets --- build/ci/community/check_all_checks.sh | 2 ++ build/ci/community/prepare_env.sh | 5 +++-- build/ci/community/wait_demo_pr_checks.sh | 11 +++++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/build/ci/community/check_all_checks.sh b/build/ci/community/check_all_checks.sh index 429975a6d..af0792460 100755 --- a/build/ci/community/check_all_checks.sh +++ b/build/ci/community/check_all_checks.sh @@ -1,6 +1,8 @@ #!/bin/bash -xe set +o pipefail +# CSI-3172 - run rad hat bot checks + did_all_checks_pass(){ community_operators_branch=$1 all_checks_passed=false diff --git a/build/ci/community/prepare_env.sh b/build/ci/community/prepare_env.sh index 03c64ae06..4643afc00 100755 --- a/build/ci/community/prepare_env.sh +++ b/build/ci/community/prepare_env.sh @@ -1,10 +1,11 @@ #!/bin/bash -xel set +o pipefail -operator_package_file_path=deploy/olm-catalog/ibm-block-csi-operator-community/ibm-block-csi-operator.package.yaml +operator_package_file_path=deploy/olm-catalog/ibm-block-csi-operator/ibm-block-csi-operator.package.yaml csv_version=`yq eval .channels[0].currentCSV $operator_package_file_path` +# CSI-3171 - handle non GA versions export csv_version=`echo ${csv_version//ibm-block-csi-operator.v}` export repository_path=~/work/$CI_REPOSITORY_NAME/$CI_REPOSITORY_NAME -export csv_file=$repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$csv_version/ibm-block-csi-operator.v$csv_version.clusterserviceversion.yaml +export csv_file=$repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$csv_version/manifests/ibm-block-csi-operator.v$csv_version.clusterserviceversion.yaml export upstream_community_operators_path=upstream-community-operators/ibm-block-csi-operator-community/$csv_version export community_operators_path=community-operators/ibm-block-csi-operator-community/$csv_version diff --git a/build/ci/community/wait_demo_pr_checks.sh b/build/ci/community/wait_demo_pr_checks.sh index 6c87d6461..c0a2f6168 100755 --- a/build/ci/community/wait_demo_pr_checks.sh +++ b/build/ci/community/wait_demo_pr_checks.sh @@ -1,22 +1,25 @@ #!/bin/bash -xe set +o pipefail +gh_pr_checks_command (){ + community_operators_branch=$1 + gh pr checks $community_operators_branch --repo $forked_community_operators_repository +} + wait_fot_checks_to_start(){ community_operators_branch=$1 forked_community_operators_repository=$2 - gh_pr_checks_command="gh pr checks $community_operators_branch --repo $forked_community_operators_repository" - while [ `$gh_pr_checks_command | grep -i pending | wc -l` -eq 0 ]; do + while [ `gh_pr_checks_command $community_operators_branch | grep -i pending | wc -l` -eq 0 ]; do sleep 1 done } wait_for_checks_to_complete(){ community_operators_branch=$1 all_tests_passed=false - gh_pr_checks_command="gh pr checks $community_operators_branch --repo $forked_community_operators_repository" repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then wait_fot_checks_to_start $community_operators_branch $forked_community_operators_repository - test_summary="$gh_pr_checks_command | grep -i summary" + test_summary="gh_pr_checks_command $community_operators_branch | grep -i summary" while [[ ! "`eval $test_summary`" =~ "pass" ]] && [[ ! "`eval $test_summary`" =~ "fail" ]]; do sleep 1 done From 1fcdb33dd3a93f2e37b5042e9ac32158c3df01ef Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Mon, 12 Jul 2021 15:04:40 +0300 Subject: [PATCH 41/51] add more permmisions to edit csv_file --- build/ci/community/create_demo_pr.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index 610193ea7..4894abf4f 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -2,6 +2,7 @@ set +o pipefail cd $(dirname $csv_file) +chmod 547 $(basename $csv_file) declare -a operator_image_fields=( ".spec.install.spec.deployments[0].spec.template.spec.containers[0].image" ".metadata.annotations.containerImage" From e4b57bfa9a6877b16ab85ecf561f6b34a6794e7b Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Tue, 13 Jul 2021 09:18:26 +0300 Subject: [PATCH 42/51] change wait for checks condition --- build/ci/community/wait_demo_pr_checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci/community/wait_demo_pr_checks.sh b/build/ci/community/wait_demo_pr_checks.sh index c0a2f6168..87e93ac85 100755 --- a/build/ci/community/wait_demo_pr_checks.sh +++ b/build/ci/community/wait_demo_pr_checks.sh @@ -9,7 +9,7 @@ gh_pr_checks_command (){ wait_fot_checks_to_start(){ community_operators_branch=$1 forked_community_operators_repository=$2 - while [ `gh_pr_checks_command $community_operators_branch | grep -i pending | wc -l` -eq 0 ]; do + while [ `gh_pr_checks_command $community_operators_branch | wc -l` -eq 0 ]; do sleep 1 done } From 75f4007cd502382146968fd8f0fc52252a75aae5 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 4 Aug 2021 10:27:57 +0300 Subject: [PATCH 43/51] add some functions --- build/ci/community/create_demo_pr.sh | 46 +++++++++++++---------- build/ci/community/prepare_env.sh | 10 ++--- build/ci/community/wait_demo_pr_checks.sh | 5 +-- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index 4894abf4f..21cb82ce9 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -1,27 +1,20 @@ #!/bin/bash -xel set +o pipefail -cd $(dirname $csv_file) -chmod 547 $(basename $csv_file) -declare -a operator_image_fields=( - ".spec.install.spec.deployments[0].spec.template.spec.containers[0].image" - ".metadata.annotations.containerImage" - ".spec.relatedImages[0].image" -) -for image_field in "${operator_image_fields[@]}" -do - yq eval "$image_field |= env(operator_image_for_test)" $(basename $csv_file) -i -done +edit_operator_image_in_csv_yaml_file (){ + cd $(dirname $csv_file) + chmod 547 $(basename $csv_file) + declare -a operator_image_fields=( + ".spec.install.spec.deployments[0].spec.template.spec.containers[0].image" + ".metadata.annotations.containerImage" + ".spec.relatedImages[0].image" + ) + for image_field in "${operator_image_fields[@]}" + do + yq eval "$image_field |= env(operator_image_for_test)" $(basename $csv_file) -i + done cd - - -echo $github_token > github_token.txt -gh auth login --with-token < github_token.txt -gh repo fork operator-framework/community-operators --clone community-operators-fork -cd community-operators-fork -git remote set-url origin https://csiblock:$github_token@github.com/csiblock/community-operators.git -git fetch upstream -git rebase upstream/master -git push origin master --force +} create_demo_pr(){ community_operators_branch=$1 @@ -40,5 +33,18 @@ create_demo_pr(){ gh pr create --title "IBM Block CSI update $cluster_kind" --repo $forked_community_operators_repository --base master --head $community_operators_branch --body "pr check" } +update_community_operators_fork (){ + echo $github_token > github_token.txt + gh auth login --with-token < github_token.txt + gh repo fork operator-framework/community-operators --clone community-operators-fork + cd community-operators-fork + git remote set-url origin https://csiblock:$github_token@github.com/csiblock/community-operators.git + git fetch upstream + git rebase upstream/master + git push origin master --force +} + +edit_operator_image_in_csv_yaml_file +update_community_operators_fork create_demo_pr $community_operators_kubernetes_branch "upstream-community-operators/" "kubernetes" create_demo_pr $community_operators_openshift_branch "community-operators/" "openshift" diff --git a/build/ci/community/prepare_env.sh b/build/ci/community/prepare_env.sh index 4643afc00..82f739cf6 100755 --- a/build/ci/community/prepare_env.sh +++ b/build/ci/community/prepare_env.sh @@ -3,11 +3,11 @@ set +o pipefail operator_package_file_path=deploy/olm-catalog/ibm-block-csi-operator/ibm-block-csi-operator.package.yaml csv_version=`yq eval .channels[0].currentCSV $operator_package_file_path` # CSI-3171 - handle non GA versions -export csv_version=`echo ${csv_version//ibm-block-csi-operator.v}` -export repository_path=~/work/$CI_REPOSITORY_NAME/$CI_REPOSITORY_NAME -export csv_file=$repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$csv_version/manifests/ibm-block-csi-operator.v$csv_version.clusterserviceversion.yaml -export upstream_community_operators_path=upstream-community-operators/ibm-block-csi-operator-community/$csv_version -export community_operators_path=community-operators/ibm-block-csi-operator-community/$csv_version +csv_version=`echo ${csv_version//ibm-block-csi-operator.v}` +repository_path=~/work/$CI_REPOSITORY_NAME/$CI_REPOSITORY_NAME +csv_file=$repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$csv_version/manifests/ibm-block-csi-operator.v$csv_version.clusterserviceversion.yaml +upstream_community_operators_path=upstream-community-operators/ibm-block-csi-operator-community/$csv_version +community_operators_path=community-operators/ibm-block-csi-operator-community/$csv_version echo "::set-output name=upstream_community_operators_path::${upstream_community_operators_path}" echo "::set-output name=community_operators_path::${community_operators_path}" diff --git a/build/ci/community/wait_demo_pr_checks.sh b/build/ci/community/wait_demo_pr_checks.sh index 87e93ac85..e329cf0b8 100755 --- a/build/ci/community/wait_demo_pr_checks.sh +++ b/build/ci/community/wait_demo_pr_checks.sh @@ -6,9 +6,8 @@ gh_pr_checks_command (){ gh pr checks $community_operators_branch --repo $forked_community_operators_repository } -wait_fot_checks_to_start(){ +wait_for_checks_to_start(){ community_operators_branch=$1 - forked_community_operators_repository=$2 while [ `gh_pr_checks_command $community_operators_branch | wc -l` -eq 0 ]; do sleep 1 done @@ -18,7 +17,7 @@ wait_for_checks_to_complete(){ all_tests_passed=false repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then - wait_fot_checks_to_start $community_operators_branch $forked_community_operators_repository + wait_for_checks_to_start $community_operators_branch test_summary="gh_pr_checks_command $community_operators_branch | grep -i summary" while [[ ! "`eval $test_summary`" =~ "pass" ]] && [[ ! "`eval $test_summary`" =~ "fail" ]]; do sleep 1 From d40871fb21ac8ec3852a75ce41eb5ff58938f6c9 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 4 Aug 2021 13:24:08 +0300 Subject: [PATCH 44/51] support new way of community operators --- .github/workflows/csi_operator_ci.yaml | 3 +++ build/ci/community/check_all_checks.sh | 7 +++--- build/ci/community/clean_demo_pr.sh | 11 ++++----- build/ci/community/create_demo_pr.sh | 27 ++++++++++++++--------- build/ci/community/wait_demo_pr_checks.sh | 17 ++++++++------ 5 files changed, 40 insertions(+), 25 deletions(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index 0c92bae53..c66bfe441 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -54,7 +54,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 env: + original_community_operators_repository: "k8s-operatorhub/community-operators" + original_community_operators_repository_prod: "redhat-openshift-ecosystem/community-operators-prod" forked_community_operators_repository: "csiblock/community-operators" + forked_community_operators_repository_prod: "csiblock/community-operators-prod" community_operators_kubernetes_branch: "ibm-block-csi-update-kubernetes-${{ github.run_number }}" community_operators_openshift_branch: "ibm-block-csi-update-openshift-${{ github.run_number }}" github_token: ${{ secrets.CSIBLOCK_GITHUB_TOKEN }} diff --git a/build/ci/community/check_all_checks.sh b/build/ci/community/check_all_checks.sh index af0792460..8cd0c0a74 100755 --- a/build/ci/community/check_all_checks.sh +++ b/build/ci/community/check_all_checks.sh @@ -5,16 +5,17 @@ set +o pipefail did_all_checks_pass(){ community_operators_branch=$1 + forked_repository=$2 all_checks_passed=false - if [ "$(gh pr checks $community_operators_branch --repo $forked_community_operators_repository | grep -iv pass | wc -l)" -eq 0 ] + if [ "$(gh pr checks $community_operators_branch --repo $forked_repository | grep -iv pass | wc -l)" -eq 0 ] then all_checks_passed=true fi echo "$all_checks_passed" } -passed_k8s_checks=$(did_all_checks_pass $community_operators_kubernetes_branch) -passed_openshift_checks=$(did_all_checks_pass $community_operators_openshift_branch) +passed_k8s_checks=$(did_all_checks_pass $community_operators_kubernetes_branch $forked_community_operators_repository) +passed_openshift_checks=$(did_all_checks_pass $community_operators_openshift_branch $forked_community_operators_repository_prod) if [ $passed_k8s_checks == "false" ] || [ $passed_openshift_checks == "false" ] then diff --git a/build/ci/community/clean_demo_pr.sh b/build/ci/community/clean_demo_pr.sh index f6209cc5b..fae8e36f3 100755 --- a/build/ci/community/clean_demo_pr.sh +++ b/build/ci/community/clean_demo_pr.sh @@ -4,13 +4,14 @@ set +o pipefail print_checks_and_delete_pr(){ community_operators_branch=$1 cluster_kind=$2 - repo_pr="gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch" + forked_repository=$3 + repo_pr="gh pr list --repo $forked_repository | grep $community_operators_branch" if [[ "`eval $repo_pr`" == *"$community_operators_branch"* ]]; then echo "The $cluster_kind checks:" - gh pr checks $community_operators_branch --repo $forked_community_operators_repository || true - gh pr close $community_operators_branch --delete-branch --repo $forked_community_operators_repository + gh pr checks $community_operators_branch --repo $forked_repository || true + gh pr close $community_operators_branch --delete-branch --repo $forked_repository fi } -print_checks_and_delete_pr $community_operators_kubernetes_branch 'kubernetes' -print_checks_and_delete_pr $community_operators_openshift_branch 'openshift' +print_checks_and_delete_pr $community_operators_kubernetes_branch 'kubernetes' $forked_community_operators_repository +print_checks_and_delete_pr $community_operators_openshift_branch 'openshift' $forked_community_operators_repository_prod diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index 21cb82ce9..cbf1bc492 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -19,10 +19,12 @@ cd - create_demo_pr(){ community_operators_branch=$1 dest_path=$2 - cluster_kind=$3 - repo_pr="gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch" + cluster_kind=$3 + forked_repository=$4 + cd $forked_repository-fork + repo_pr="gh pr list --repo $forked_repository | grep $community_operators_branch" if [[ "`eval $repo_pr`" == *"$community_operators_branch"* ]]; then - gh pr close $community_operators_branch --delete-branch --repo $forked_community_operators_repository + gh pr close $community_operators_branch --delete-branch --repo $forked_repository fi git checkout master git checkout -b $community_operators_branch @@ -30,21 +32,26 @@ create_demo_pr(){ git add . git commit --signoff -m "build number $github_build_number $cluster_kind" git push origin $community_operators_branch - gh pr create --title "IBM Block CSI update $cluster_kind" --repo $forked_community_operators_repository --base master --head $community_operators_branch --body "pr check" + gh pr create --title "IBM Block CSI update $cluster_kind" --repo $forked_repository --base master --head $community_operators_branch --body "pr check" + cd .. } update_community_operators_fork (){ + forked_repository=$1 + original_repository=$2 echo $github_token > github_token.txt gh auth login --with-token < github_token.txt - gh repo fork operator-framework/community-operators --clone community-operators-fork - cd community-operators-fork - git remote set-url origin https://csiblock:$github_token@github.com/csiblock/community-operators.git + gh repo fork $original_repository --clone $forked_repository-fork + cd $forked_repository-fork + git remote set-url origin https://csiblock:$github_token@github.com/$forked_repository.git git fetch upstream git rebase upstream/master git push origin master --force + cd .. } edit_operator_image_in_csv_yaml_file -update_community_operators_fork -create_demo_pr $community_operators_kubernetes_branch "upstream-community-operators/" "kubernetes" -create_demo_pr $community_operators_openshift_branch "community-operators/" "openshift" +update_community_operators_fork $forked_community_operators_repository $original_community_operators_repository +update_community_operators_fork $forked_community_operators_repository_prod $original_community_operators_repository_prod +create_demo_pr $community_operators_kubernetes_branch "operators/" "kubernetes" $forked_community_operators_repository +create_demo_pr $community_operators_openshift_branch "operators/" "openshift" $forked_community_operators_repository_prod diff --git a/build/ci/community/wait_demo_pr_checks.sh b/build/ci/community/wait_demo_pr_checks.sh index e329cf0b8..8c246eb01 100755 --- a/build/ci/community/wait_demo_pr_checks.sh +++ b/build/ci/community/wait_demo_pr_checks.sh @@ -3,27 +3,30 @@ set +o pipefail gh_pr_checks_command (){ community_operators_branch=$1 - gh pr checks $community_operators_branch --repo $forked_community_operators_repository + forked_repository=$2 + gh pr checks $community_operators_branch --repo $forked_repository } wait_for_checks_to_start(){ community_operators_branch=$1 - while [ `gh_pr_checks_command $community_operators_branch | wc -l` -eq 0 ]; do + forked_repository=$2 + while [ `gh_pr_checks_command $community_operators_branch $forked_repository | wc -l` -eq 0 ]; do sleep 1 done } wait_for_checks_to_complete(){ community_operators_branch=$1 + forked_repository=$2 all_tests_passed=false - repo_pr=`gh pr list --repo $forked_community_operators_repository | grep $community_operators_branch` + repo_pr=`gh pr list --repo $forked_repository | grep $community_operators_branch` if [[ "$repo_pr" == *"$community_operators_branch"* ]]; then - wait_for_checks_to_start $community_operators_branch - test_summary="gh_pr_checks_command $community_operators_branch | grep -i summary" + wait_for_checks_to_start $community_operators_branch $forked_repository + test_summary="gh_pr_checks_command $community_operators_branch $forked_repository | grep -i summary" while [[ ! "`eval $test_summary`" =~ "pass" ]] && [[ ! "`eval $test_summary`" =~ "fail" ]]; do sleep 1 done fi } -wait_for_checks_to_complete $community_operators_kubernetes_branch -wait_for_checks_to_complete $community_operators_openshift_branch +wait_for_checks_to_complete $community_operators_kubernetes_branch $forked_community_operators_repository +wait_for_checks_to_complete $community_operators_openshift_branch $forked_community_operators_repository_prod From e9da769060bcf346f8650e4a1a044d29049a7a83 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 4 Aug 2021 14:18:20 +0300 Subject: [PATCH 45/51] add cd - --- build/ci/community/create_demo_pr.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index cbf1bc492..5535bc2c6 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -33,7 +33,7 @@ create_demo_pr(){ git commit --signoff -m "build number $github_build_number $cluster_kind" git push origin $community_operators_branch gh pr create --title "IBM Block CSI update $cluster_kind" --repo $forked_repository --base master --head $community_operators_branch --body "pr check" - cd .. + cd - } update_community_operators_fork (){ @@ -47,7 +47,7 @@ update_community_operators_fork (){ git fetch upstream git rebase upstream/master git push origin master --force - cd .. + cd - } edit_operator_image_in_csv_yaml_file From 3e2f6d2165e0491eb02a34956dca0b0a0bc8437b Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Wed, 4 Aug 2021 14:25:43 +0300 Subject: [PATCH 46/51] change master branch to main --- build/ci/community/create_demo_pr.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index 5535bc2c6..048768659 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -26,13 +26,13 @@ create_demo_pr(){ if [[ "`eval $repo_pr`" == *"$community_operators_branch"* ]]; then gh pr close $community_operators_branch --delete-branch --repo $forked_repository fi - git checkout master + git checkout main git checkout -b $community_operators_branch yes | cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/ $dest_path git add . git commit --signoff -m "build number $github_build_number $cluster_kind" git push origin $community_operators_branch - gh pr create --title "IBM Block CSI update $cluster_kind" --repo $forked_repository --base master --head $community_operators_branch --body "pr check" + gh pr create --title "IBM Block CSI update $cluster_kind" --repo $forked_repository --base main --head $community_operators_branch --body "pr check" cd - } @@ -45,8 +45,8 @@ update_community_operators_fork (){ cd $forked_repository-fork git remote set-url origin https://csiblock:$github_token@github.com/$forked_repository.git git fetch upstream - git rebase upstream/master - git push origin master --force + git rebase upstream/main + git push origin main --force cd - } From d4a0f6baad4cc53a98d62e09daa87c4608472d99 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Sun, 3 Oct 2021 15:30:17 +0300 Subject: [PATCH 47/51] remove file from function Signed-off-by: matancarmeli7 --- build/ci/community/create_demo_pr.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index 048768659..41f80e36f 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -1,7 +1,7 @@ #!/bin/bash -xel set +o pipefail -edit_operator_image_in_csv_yaml_file (){ +edit_operator_image_in_csv_yaml (){ cd $(dirname $csv_file) chmod 547 $(basename $csv_file) declare -a operator_image_fields=( @@ -50,7 +50,7 @@ update_community_operators_fork (){ cd - } -edit_operator_image_in_csv_yaml_file +edit_operator_image_in_csv_yaml update_community_operators_fork $forked_community_operators_repository $original_community_operators_repository update_community_operators_fork $forked_community_operators_repository_prod $original_community_operators_repository_prod create_demo_pr $community_operators_kubernetes_branch "operators/" "kubernetes" $forked_community_operators_repository From a13fab7e55450b1362b100092498e08df144ffa3 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Sun, 3 Oct 2021 16:17:09 +0300 Subject: [PATCH 48/51] copy only the latest csv Signed-off-by: matancarmeli7 --- .github/workflows/csi_operator_ci.yaml | 3 +++ build/ci/community/create_demo_pr.sh | 2 +- build/ci/community/setup_ci_dependencies.sh | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/csi_operator_ci.yaml b/.github/workflows/csi_operator_ci.yaml index c66bfe441..475adc34b 100644 --- a/.github/workflows/csi_operator_ci.yaml +++ b/.github/workflows/csi_operator_ci.yaml @@ -26,6 +26,7 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - name: Setup dependencies + id: setup_dependencies run: | build/ci/community/setup_ci_dependencies.sh - name: Save dependencies files @@ -42,6 +43,7 @@ jobs: run: | build/ci/community/prepare_env.sh outputs: + image_branch_tag: "${{ steps.setup_dependencies.outputs.image_branch_tag }}" community_operators_path: "${{ steps.environment_setup.outputs.community_operators_path }}" upstream_community_operators_path: "${{ steps.environment_setup.outputs.upstream_community_operators_path }}" repository_path: "${{ steps.environment_setup.outputs.repository_path }}" @@ -65,6 +67,7 @@ jobs: operator_image_for_test: ${{ needs.prepare_env.outputs.operator_image_for_test }} csv_file: ${{ needs.prepare_env.outputs.csv_file }} github_build_number: ${{ github.run_number }} + latest_operator_version: ${{ needs.prepare_env.outputs.image_branch_tag }} steps: - name: Checkout code uses: actions/checkout@v2 diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index 41f80e36f..4fa408924 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -28,7 +28,7 @@ create_demo_pr(){ fi git checkout main git checkout -b $community_operators_branch - yes | cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/ $dest_path + yes | cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$latest_operator_version $dest_path/ibm-block-csi-operator-community/test git add . git commit --signoff -m "build number $github_build_number $cluster_kind" git push origin $community_operators_branch diff --git a/build/ci/community/setup_ci_dependencies.sh b/build/ci/community/setup_ci_dependencies.sh index 28ea792fa..c6bdcd111 100755 --- a/build/ci/community/setup_ci_dependencies.sh +++ b/build/ci/community/setup_ci_dependencies.sh @@ -10,3 +10,8 @@ sudo apt-get install gh git config --global user.email csi.block1@il.ibm.com git config --global user.name csiblock + +image_version=`cat version/version.go | grep -i driverversion | awk -F = '{print $2}'` +image_version=`echo ${image_version//\"}` + +echo "::set-output name=image_branch_tag::${image_branch_tag}" \ No newline at end of file From 613014a14ba4e6338ce478b1cf96d515e7cb66ca Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Sun, 3 Oct 2021 16:26:25 +0300 Subject: [PATCH 49/51] fix image_version Signed-off-by: matancarmeli7 --- build/ci/community/create_demo_pr.sh | 2 +- build/ci/community/setup_ci_dependencies.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index 4fa408924..def9d2264 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -28,7 +28,7 @@ create_demo_pr(){ fi git checkout main git checkout -b $community_operators_branch - yes | cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$latest_operator_version $dest_path/ibm-block-csi-operator-community/test + yes | cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$latest_operator_version/ $dest_path/ibm-block-csi-operator-community/test git add . git commit --signoff -m "build number $github_build_number $cluster_kind" git push origin $community_operators_branch diff --git a/build/ci/community/setup_ci_dependencies.sh b/build/ci/community/setup_ci_dependencies.sh index c6bdcd111..1359a1404 100755 --- a/build/ci/community/setup_ci_dependencies.sh +++ b/build/ci/community/setup_ci_dependencies.sh @@ -14,4 +14,4 @@ git config --global user.name csiblock image_version=`cat version/version.go | grep -i driverversion | awk -F = '{print $2}'` image_version=`echo ${image_version//\"}` -echo "::set-output name=image_branch_tag::${image_branch_tag}" \ No newline at end of file +echo "::set-output name=image_branch_tag::${image_version}" \ No newline at end of file From e25ec2b02fdbe50c6422178ff6c0b4ed568dafba Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Mon, 4 Oct 2021 09:56:40 +0300 Subject: [PATCH 50/51] remove test directory Signed-off-by: matancarmeli7 --- build/ci/community/create_demo_pr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci/community/create_demo_pr.sh b/build/ci/community/create_demo_pr.sh index def9d2264..fafaced9d 100755 --- a/build/ci/community/create_demo_pr.sh +++ b/build/ci/community/create_demo_pr.sh @@ -28,7 +28,7 @@ create_demo_pr(){ fi git checkout main git checkout -b $community_operators_branch - yes | cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$latest_operator_version/ $dest_path/ibm-block-csi-operator-community/test + yes | cp -r $repository_path/deploy/olm-catalog/ibm-block-csi-operator-community/$latest_operator_version/ $dest_path/ibm-block-csi-operator-community git add . git commit --signoff -m "build number $github_build_number $cluster_kind" git push origin $community_operators_branch From 7ff7b636f472c4dd36396e46ec251c3c0984f651 Mon Sep 17 00:00:00 2001 From: matancarmeli7 Date: Mon, 4 Oct 2021 10:55:44 +0300 Subject: [PATCH 51/51] change comment on redhat Signed-off-by: matancarmeli7 --- build/ci/community/check_all_checks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci/community/check_all_checks.sh b/build/ci/community/check_all_checks.sh index 8cd0c0a74..e011005d4 100755 --- a/build/ci/community/check_all_checks.sh +++ b/build/ci/community/check_all_checks.sh @@ -1,7 +1,7 @@ #!/bin/bash -xe set +o pipefail -# CSI-3172 - run rad hat bot checks +# CSI-3172 - run Red Hat bot checks did_all_checks_pass(){ community_operators_branch=$1