From b123dbf2b568873bae8477c9a445caf43d65f414 Mon Sep 17 00:00:00 2001 From: Andrey Velichkevich Date: Mon, 28 Nov 2022 14:37:15 +0000 Subject: [PATCH] [Test] Reduce Katib GitHub Action Runs (#2036) * [Test] Reduce Katib GitHub Action Runs * Add cancel-in-progress flag * Use single job for Charmed Katib * Add cancel-in-progress for all actions except publish * Bump ubuntu to 20.04 for Charmed tests --- ...-test.yaml => e2e-test-darts-cifar10.yaml} | 9 +++-- ...e-test.yaml => e2e-test-enas-cifar10.yaml} | 9 +++-- ...2e-test.yaml => e2e-test-mxnet-mnist.yaml} | 15 ++++---- ...-test.yaml => e2e-test-pytorch-mnist.yaml} | 9 +++-- ...e2e-test.yaml => e2e-test-simple-pbt.yaml} | 9 +++-- ... => e2e-test-tf-mnist-with-summaries.yaml} | 9 +++-- .../e2e-test-ui-random-search-postgres.yaml | 38 +++++++++++++++++++ .github/workflows/katib-ui-e2e-test.yaml | 30 --------------- .github/workflows/lint.yaml | 22 ----------- .../workflows/publish-algorithm-images.yaml | 4 +- .github/workflows/publish-core-images.yaml | 8 +--- .github/workflows/publish-katib-ui-image.yaml | 17 --------- .github/workflows/publish-trial-images.yaml | 4 +- .github/workflows/test-charmed-katib.yaml | 38 +++++++------------ .github/workflows/test-go.yaml | 9 +++-- .github/workflows/test-lint.yaml | 30 +++++++++++++++ .github/workflows/test-node.yaml | 5 ++- .github/workflows/test-python.yaml | 9 +++-- .github/workflows/test-shell-script.yaml | 21 ---------- 19 files changed, 137 insertions(+), 158 deletions(-) rename .github/workflows/{darts-cifar10-e2e-test.yaml => e2e-test-darts-cifar10.yaml} (86%) rename .github/workflows/{enas-cifar10-e2e-test.yaml => e2e-test-enas-cifar10.yaml} (86%) rename .github/workflows/{mxnet-mnist-e2e-test.yaml => e2e-test-mxnet-mnist.yaml} (73%) rename .github/workflows/{pytorch-mnist-e2e-test.yaml => e2e-test-pytorch-mnist.yaml} (87%) rename .github/workflows/{simple-pbt-e2e-test.yaml => e2e-test-simple-pbt.yaml} (86%) rename .github/workflows/{tf-mnist-with-summaries-e2e-test.yaml => e2e-test-tf-mnist-with-summaries.yaml} (86%) create mode 100644 .github/workflows/e2e-test-ui-random-search-postgres.yaml delete mode 100644 .github/workflows/katib-ui-e2e-test.yaml delete mode 100755 .github/workflows/lint.yaml delete mode 100644 .github/workflows/publish-katib-ui-image.yaml create mode 100755 .github/workflows/test-lint.yaml delete mode 100644 .github/workflows/test-shell-script.yaml diff --git a/.github/workflows/darts-cifar10-e2e-test.yaml b/.github/workflows/e2e-test-darts-cifar10.yaml similarity index 86% rename from .github/workflows/darts-cifar10-e2e-test.yaml rename to .github/workflows/e2e-test-darts-cifar10.yaml index 629fce4f378..6e65767d25b 100644 --- a/.github/workflows/darts-cifar10-e2e-test.yaml +++ b/.github/workflows/e2e-test-darts-cifar10.yaml @@ -1,8 +1,13 @@ name: E2E Test with darts-cnn-cifar10 + on: pull_request: paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' + - "pkg/new-ui/v1beta1/frontend/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -26,7 +31,6 @@ jobs: experiments: ${{ matrix.experiments }} # Comma Delimited trial-images: darts-cnn-cifar10-cpu - database-type: ${{ matrix.database-type }} strategy: fail-fast: false @@ -34,4 +38,3 @@ jobs: kubernetes-version: ["v1.23.13", "v1.24.7", "v1.25.3"] # Comma Delimited experiments: ["darts-cpu"] - database-type: ["mysql", "postgres"] diff --git a/.github/workflows/enas-cifar10-e2e-test.yaml b/.github/workflows/e2e-test-enas-cifar10.yaml similarity index 86% rename from .github/workflows/enas-cifar10-e2e-test.yaml rename to .github/workflows/e2e-test-enas-cifar10.yaml index fd9f243f89b..d3e6be6e7b2 100644 --- a/.github/workflows/enas-cifar10-e2e-test.yaml +++ b/.github/workflows/e2e-test-enas-cifar10.yaml @@ -1,8 +1,13 @@ name: E2E Test with enas-cnn-cifar10 + on: pull_request: paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' + - "pkg/new-ui/v1beta1/frontend/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -26,7 +31,6 @@ jobs: experiments: ${{ matrix.experiments }} # Comma Delimited trial-images: enas-cnn-cifar10-cpu - database-type: ${{ matrix.database-type }} strategy: fail-fast: false @@ -34,4 +38,3 @@ jobs: kubernetes-version: ["v1.23.13", "v1.24.7", "v1.25.3"] # Comma Delimited experiments: ["enas-cpu"] - database-type: ["mysql", "postgres"] diff --git a/.github/workflows/mxnet-mnist-e2e-test.yaml b/.github/workflows/e2e-test-mxnet-mnist.yaml similarity index 73% rename from .github/workflows/mxnet-mnist-e2e-test.yaml rename to .github/workflows/e2e-test-mxnet-mnist.yaml index 828379bbb3f..ad3a440511e 100644 --- a/.github/workflows/mxnet-mnist-e2e-test.yaml +++ b/.github/workflows/e2e-test-mxnet-mnist.yaml @@ -1,8 +1,13 @@ name: E2E Test with mxnet-mnist + on: pull_request: paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' + - "pkg/new-ui/v1beta1/frontend/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -26,7 +31,6 @@ jobs: experiments: ${{ matrix.experiments }} # Comma Delimited trial-images: mxnet-mnist - database-type: ${{ matrix.database-type }} strategy: fail-fast: false @@ -35,9 +39,6 @@ jobs: # Comma Delimited experiments: # suggestion-hyperopt - - "random,tpe,never-resume" - - "median-stop,from-volume-resume" + - "never-resume,from-volume-resume,median-stop" # others - - "grid,bayesian-optimization,tpe" - - "multivariate-tpe,cma-es,hyperband" - database-type: ["mysql", "postgres"] + - "grid,bayesian-optimization,tpe,multivariate-tpe,cma-es,hyperband" diff --git a/.github/workflows/pytorch-mnist-e2e-test.yaml b/.github/workflows/e2e-test-pytorch-mnist.yaml similarity index 87% rename from .github/workflows/pytorch-mnist-e2e-test.yaml rename to .github/workflows/e2e-test-pytorch-mnist.yaml index c0c79962a61..8a16ac86984 100644 --- a/.github/workflows/pytorch-mnist-e2e-test.yaml +++ b/.github/workflows/e2e-test-pytorch-mnist.yaml @@ -1,8 +1,13 @@ name: E2E Test with pytorch-mnist + on: pull_request: paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' + - "pkg/new-ui/v1beta1/frontend/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -27,7 +32,6 @@ jobs: training-operator: true # Comma Delimited trial-images: pytorch-mnist-cpu - database-type: ${{ matrix.database-type }} strategy: fail-fast: false @@ -37,4 +41,3 @@ jobs: experiments: - "file-metrics-collector,pytorchjob-mnist" - "median-stop-with-json-format,file-metrics-collector-with-json-format" - database-type: ["mysql", "postgres"] diff --git a/.github/workflows/simple-pbt-e2e-test.yaml b/.github/workflows/e2e-test-simple-pbt.yaml similarity index 86% rename from .github/workflows/simple-pbt-e2e-test.yaml rename to .github/workflows/e2e-test-simple-pbt.yaml index 40b299c68e5..2bc01b8b389 100644 --- a/.github/workflows/simple-pbt-e2e-test.yaml +++ b/.github/workflows/e2e-test-simple-pbt.yaml @@ -1,8 +1,13 @@ name: E2E Test with simple-pbt + on: pull_request: paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' + - "pkg/new-ui/v1beta1/frontend/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -26,7 +31,6 @@ jobs: experiments: ${{ matrix.experiments }} # Comma Delimited trial-images: simple-pbt - database-type: ${{ matrix.database-type }} strategy: fail-fast: false @@ -35,4 +39,3 @@ jobs: kubernetes-version: ["v1.23.13", "v1.24.7", "v1.25.3"] # Comma Delimited experiments: ["simple-pbt"] - database-type: ["mysql", "postgres"] diff --git a/.github/workflows/tf-mnist-with-summaries-e2e-test.yaml b/.github/workflows/e2e-test-tf-mnist-with-summaries.yaml similarity index 86% rename from .github/workflows/tf-mnist-with-summaries-e2e-test.yaml rename to .github/workflows/e2e-test-tf-mnist-with-summaries.yaml index da70b0f6690..74c28930ffe 100644 --- a/.github/workflows/tf-mnist-with-summaries-e2e-test.yaml +++ b/.github/workflows/e2e-test-tf-mnist-with-summaries.yaml @@ -1,8 +1,13 @@ name: E2E Test with tf-mnist-with-summaries + on: pull_request: paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' + - "pkg/new-ui/v1beta1/frontend/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -27,7 +32,6 @@ jobs: training-operator: true # Comma Delimited trial-images: tf-mnist-with-summaries - database-type: ${{ matrix.database-type }} strategy: fail-fast: false @@ -35,4 +39,3 @@ jobs: kubernetes-version: ["v1.23.13", "v1.24.7", "v1.25.3"] # Comma Delimited experiments: ["tfjob-mnist-with-summaries"] - database-type: ["mysql", "postgres"] diff --git a/.github/workflows/e2e-test-ui-random-search-postgres.yaml b/.github/workflows/e2e-test-ui-random-search-postgres.yaml new file mode 100644 index 00000000000..89d06dcae27 --- /dev/null +++ b/.github/workflows/e2e-test-ui-random-search-postgres.yaml @@ -0,0 +1,38 @@ +name: E2E Test with Katib UI, random search, and postgres + +on: + - pull_request + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + e2e: + runs-on: ubuntu-20.04 + timeout-minutes: 120 + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Test Env + uses: ./.github/workflows/template-setup-e2e-test + with: + kubernetes-version: ${{ matrix.kubernetes-version }} + + - name: Run e2e test with ${{ matrix.experiments }} experiments + uses: ./.github/workflows/template-e2e-test + with: + experiments: random + # Comma Delimited + trial-images: mxnet-mnist + katib-ui: true + database-type: postgres + + strategy: + fail-fast: false + matrix: + kubernetes-version: ["v1.23.13", "v1.24.7", "v1.25.3"] diff --git a/.github/workflows/katib-ui-e2e-test.yaml b/.github/workflows/katib-ui-e2e-test.yaml deleted file mode 100644 index db5ae36936a..00000000000 --- a/.github/workflows/katib-ui-e2e-test.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: E2E Test for katib-ui -on: - - pull_request - -env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - -jobs: - e2e: - runs-on: ubuntu-20.04 - timeout-minutes: 120 - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Test Env - uses: ./.github/workflows/template-setup-e2e-test - with: - kubernetes-version: ${{ matrix.kubernetes-version }} - - - name: Set Up Minikube Cluster - run: ./test/e2e/v1beta1/scripts/gh-actions/setup-minikube.sh true - - - name: Start Katib - run: ./test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh true false - - strategy: - fail-fast: false - matrix: - kubernetes-version: ["v1.23.13", "v1.24.7", "v1.25.3"] diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml deleted file mode 100755 index c4909740b2d..00000000000 --- a/.github/workflows/lint.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: Lint YAML files - -on: - - push - - pull_request - -jobs: - lint: - name: Lint - runs-on: ubuntu-latest - - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - - name: Check YAML - run: make yamllint diff --git a/.github/workflows/publish-algorithm-images.yaml b/.github/workflows/publish-algorithm-images.yaml index 580c7ef55d9..0f53ab8e46e 100644 --- a/.github/workflows/publish-algorithm-images.yaml +++ b/.github/workflows/publish-algorithm-images.yaml @@ -2,11 +2,9 @@ name: Publish AutoML Algorithm Images on: push: - paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' pull_request: paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' + - "pkg/new-ui/v1beta1/frontend/**" jobs: algorithm: diff --git a/.github/workflows/publish-core-images.yaml b/.github/workflows/publish-core-images.yaml index b6c17109e9e..750ab03c99e 100644 --- a/.github/workflows/publish-core-images.yaml +++ b/.github/workflows/publish-core-images.yaml @@ -1,12 +1,8 @@ name: Publish Katib Core Images on: - push: - paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' - pull_request: - paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' + - push + - pull_request jobs: core: diff --git a/.github/workflows/publish-katib-ui-image.yaml b/.github/workflows/publish-katib-ui-image.yaml deleted file mode 100644 index 148761ff69a..00000000000 --- a/.github/workflows/publish-katib-ui-image.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: Publish Katib UI Image - -on: - - push - - pull_request - -jobs: - ui: - name: Publish Image - uses: ./.github/workflows/build-and-publish-images.yaml - with: - component-name: katib-ui - platforms: linux/amd64,linux/arm64 - dockerfile: cmd/new-ui/v1beta1/Dockerfile - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/publish-trial-images.yaml b/.github/workflows/publish-trial-images.yaml index 5e3172555e7..0e810288f3e 100644 --- a/.github/workflows/publish-trial-images.yaml +++ b/.github/workflows/publish-trial-images.yaml @@ -2,11 +2,9 @@ name: Publish Trial Images on: push: - paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' pull_request: paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' + - "pkg/new-ui/v1beta1/frontend/**" jobs: trial: diff --git a/.github/workflows/test-charmed-katib.yaml b/.github/workflows/test-charmed-katib.yaml index 88ce3488526..7d1d7b5b871 100644 --- a/.github/workflows/test-charmed-katib.yaml +++ b/.github/workflows/test-charmed-katib.yaml @@ -1,17 +1,18 @@ name: Charmed Katib on: - push: - paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' pull_request: paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' + - "pkg/new-ui/v1beta1/frontend/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - lint: - name: Lint - runs-on: ubuntu-latest + charmed: + name: Lint and Test + runs-on: ubuntu-20.04 steps: - name: Check out code @@ -21,8 +22,13 @@ jobs: run: | set -eux sudo apt update + sudo apt install -y python3-pip sudo apt install python3-setuptools sudo pip3 install black flake8 + sudo snap install juju --classic + sudo snap install juju-bundle --classic + sudo snap install juju-wait --classic + sudo pip3 install charmcraft==1.3.1 - name: Check black run: black --check operators/*/src @@ -30,29 +36,11 @@ jobs: - name: Check flake8 run: cd operators && flake8 ./katib*/src - build: - name: Test - runs-on: ubuntu-latest - - steps: - - name: Check out repo - uses: actions/checkout@v3 - - uses: balchua/microk8s-actions@v0.2.2 with: channel: "1.21/stable" addons: '["dns", "storage", "rbac"]' - - name: Install dependencies - run: | - set -eux - sudo apt update - sudo apt install -y python3-pip - sudo snap install juju --classic - sudo snap install juju-bundle --classic - sudo snap install juju-wait --classic - sudo pip3 install charmcraft==1.3.1 - - name: Set Up Docker Buildx uses: docker/setup-buildx-action@v2 diff --git a/.github/workflows/test-go.yaml b/.github/workflows/test-go.yaml index aab1e48e249..6f162759556 100644 --- a/.github/workflows/test-go.yaml +++ b/.github/workflows/test-go.yaml @@ -1,12 +1,13 @@ name: Go Test on: - push: - paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' pull_request: paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' + - "pkg/new-ui/v1beta1/frontend/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: generatetests: diff --git a/.github/workflows/test-lint.yaml b/.github/workflows/test-lint.yaml new file mode 100755 index 00000000000..efb6698be13 --- /dev/null +++ b/.github/workflows/test-lint.yaml @@ -0,0 +1,30 @@ +name: Lint Files + +on: + pull_request: + paths-ignore: + - "pkg/new-ui/v1beta1/frontend/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Check YAML files + run: make yamllint + + - name: Check shell scripts + run: make shellcheck diff --git a/.github/workflows/test-node.yaml b/.github/workflows/test-node.yaml index 0254f5150db..2c445588332 100644 --- a/.github/workflows/test-node.yaml +++ b/.github/workflows/test-node.yaml @@ -1,9 +1,12 @@ name: Frontend Test on: - - push - pull_request +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: test: name: Test diff --git a/.github/workflows/test-python.yaml b/.github/workflows/test-python.yaml index 22e5480e183..074ea60e274 100644 --- a/.github/workflows/test-python.yaml +++ b/.github/workflows/test-python.yaml @@ -1,12 +1,13 @@ name: Python Test on: - push: - paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' pull_request: paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' + - "pkg/new-ui/v1beta1/frontend/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: test: diff --git a/.github/workflows/test-shell-script.yaml b/.github/workflows/test-shell-script.yaml deleted file mode 100644 index b456c2da378..00000000000 --- a/.github/workflows/test-shell-script.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: Shellcheck - -on: - push: - paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' - pull_request: - paths-ignore: - - 'pkg/new-ui/v1beta1/frontend/**' - -jobs: - test: - name: Test - runs-on: ubuntu-latest - - steps: - - name: Check out code - uses: actions/checkout@v3 - - - name: Run shellcheck - run: make shellcheck