Skip to content
E2E Tests

Enchance ci env #122

Sign in to view logs

Enchance ci env

Enchance ci env #122

Workflow file for this run

.github/workflows/e2e-test.yml at 542dfb8
name: E2E Tests
on:
pull_request:
branches: [master, release*]
paths:
- "**"
- "!.github/**"
- "!docs/**"
- "!**.md"
- ".github/workflows/e2e-test.yml"
workflow_dispatch:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
kserve-image-build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Build KServe images
run: |
./test/scripts/gh-actions/build-images.sh
docker image ls
cat ./config/overlays/test/configmap/inferenceservice.yaml
- name: Upload controller image
uses: ishworkh/[email protected]
with:
image: "kserve/kserve-controller:${{ github.sha }}"
- name: Upload agent image
uses: ishworkh/[email protected]
with:
image: "kserve/agent:${{ github.sha }}"
- name: Upload storage initializer image
uses: ishworkh/[email protected]
with:
image: "kserve/storage-initializer:${{ github.sha }}"
- name: Upload router image
uses: ishworkh/[email protected]
with:
image: "kserve/router:${{ github.sha }}"
poetry-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.9"
- name: Install Poetry and version plugin
run: ./test/scripts/gh-actions/setup-poetry.sh
- name: Check poetry lock file consistency
run: ./test/scripts/gh-actions/check-poetry-lockfile.sh
predictor-runtime-build:
runs-on: ubuntu-latest
needs: [poetry-check]
steps:
- uses: actions/checkout@v3
- name: Free-up disk space
uses: ./.github/actions/free-up-disk-space
- name: Build runtime server images
run: |
sudo mkdir -p /mnt/docker-images
sudo chown -R $USER /mnt/docker-images
./test/scripts/gh-actions/build-server-runtimes.sh predictor,transformer
docker image ls
cat ./config/overlays/test/configmap/inferenceservice.yaml
- name: Upload sklearn artifact
uses: actions/upload-artifact@v3
with:
name: sklearn
path: /mnt/sklearn-${GITHUB_SHA}.tar
- name: Upload xgb server image
uses: actions/upload-artifact@v3
with:
name: sklearn
path: /mnt/xgb-${GITHUB_SHA}.tar
- name: Upload lgb server image
uses: actions/upload-artifact@v3
with:
name: sklearn
path: /mnt/lgb-${GITHUB_SHA}.tar
- name: Upload pmml server image
uses: actions/upload-artifact@v3
with:
name: sklearn
path: /mnt/pmml-${GITHUB_SHA}.tar
- name: Upload paddle image
uses: actions/upload-artifact@v3
with:
name: sklearn
path: /mnt/paddle-${GITHUB_SHA}.tar
- name: Upload image transformer image
uses: actions/upload-artifact@v3
with:
name: sklearn
path: /mnt/image-transformer-${GITHUB_SHA}.tar
- name: Upload custom model grpc image
uses: actions/upload-artifact@v3
with:
name: sklearn
path: /mnt/custom-model-${GITHUB_SHA}.tar
- name: Upload custom model transformer grpc image
uses: actions/upload-artifact@v3
with:
name: sklearn
path: /mnt/custom-transformer-grpc-${GITHUB_SHA}.tar
explainer-runtime-build:
runs-on: ubuntu-latest
needs: [poetry-check]
steps:
- uses: actions/checkout@v3
- name: Build runtime server images
run: |
./test/scripts/gh-actions/build-server-runtimes.sh explainer
docker image ls
cat ./config/overlays/test/configmap/inferenceservice.yaml
- name: Upload alibi image
uses: ishworkh/[email protected]
with:
image: "kserve/alibi-explainer:${{ github.sha }}"
- name: Upload art explainer image
uses: ishworkh/[email protected]
with:
image: "kserve/art-explainer:${{ github.sha }}"
graph-tests-images-build:
runs-on: ubuntu-latest
needs: [poetry-check]
steps:
- uses: actions/checkout@v3
- name: Free-up disk space
uses: ./.github/actions/free-up-disk-space
- name: Build test images needed for graph tests
run: |
./test/scripts/gh-actions/build-graph-tests-images.sh
docker image ls
- name: Upload success_200_isvc predictor image
uses: ishworkh/[email protected]
with:
image: "kserve/success-200-isvc:${{ github.sha }}"
- name: Upload error_404_isvc predictor image
uses: ishworkh/[email protected]
with:
image: "kserve/error-404-isvc:${{ github.sha }}"
test-fast:
runs-on: ubuntu-latest
needs:
[
kserve-image-build,
predictor-runtime-build,
explainer-runtime-build,
graph-tests-images-build,
]
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup Go
uses: actions/setup-go@v4
with:
go-version: "1.21"
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.9"
- name: Free-up disk space
uses: ./.github/actions/free-up-disk-space
- name: Setup Minikube
uses: ./.github/actions/minikube-setup
- name: Download base images
uses: ./.github/actions/base-download
- name: KServe dependency setup
uses: ./.github/actions/kserve-dep-setup
- name: Download sklearn server image
uses: ishworkh/[email protected]
with:
image: "kserve/sklearnserver:${{ github.sha }}"
- name: Download xgb server image
uses: ishworkh/[email protected]
with:
image: "kserve/xgbserver:${{ github.sha }}"
- name: Download lgb server image
uses: ishworkh/[email protected]
with:
image: "kserve/lgbserver:${{ github.sha }}"
- name: Download pmml image
uses: ishworkh/[email protected]
with:
image: "kserve/pmmlserver:${{ github.sha }}"
- name: Download transformer image
uses: ishworkh/[email protected]
with:
image: "kserve/image-transformer:${{ github.sha }}"
- name: Download success_200_isvc predictor image
uses: ishworkh/[email protected]
with:
image: "kserve/success-200-isvc:${{ github.sha }}"
- name: Download error_404_isvc predictor image
uses: ishworkh/[email protected]
with:
image: "kserve/error-404-isvc:${{ github.sha }}"
- name: Install Poetry and version plugin
run: ./test/scripts/gh-actions/setup-poetry.sh
- name: Install KServe
run: |
./test/scripts/gh-actions/setup-kserve.sh
kubectl get pods -n kserve
kubectl describe pods -n kserve
- name: Run all E2E tests except graph
timeout-minutes: 60
run: |
./test/scripts/gh-actions/run-e2e-tests.sh "fast or pmml" "2"
- name: Run E2E tests for graph
timeout-minutes: 120
run: |
./test/scripts/gh-actions/run-e2e-tests.sh "graph" "2"
- name: Check system status
if: always()
run: |
./test/scripts/gh-actions/status-check.sh
test-path-based-routing:
runs-on: ubuntu-latest
needs:
[
kserve-image-build,
predictor-runtime-build,
explainer-runtime-build,
graph-tests-images-build,
]
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup Go
uses: actions/setup-go@v4
with:
go-version: "1.21"
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.9"
- name: Free-up disk space
uses: ./.github/actions/free-up-disk-space
- name: Setup Minikube
uses: ./.github/actions/minikube-setup
- name: Download base images
uses: ./.github/actions/base-download
- name: KServe dependency setup
uses: ./.github/actions/kserve-dep-setup
- name: Download sklearn server image
uses: ishworkh/[email protected]
with:
image: "kserve/sklearnserver:${{ github.sha }}"
- name: Download xgb server image
uses: ishworkh/[email protected]
with:
image: "kserve/xgbserver:${{ github.sha }}"
- name: Download lgb server image
uses: ishworkh/[email protected]
with:
image: "kserve/lgbserver:${{ github.sha }}"
- name: Download transformer image
uses: ishworkh/[email protected]
with:
image: "kserve/image-transformer:${{ github.sha }}"
- name: Install Poetry and version plugin
run: ./test/scripts/gh-actions/setup-poetry.sh
- name: Install KServe
run: |
./test/scripts/gh-actions/setup-kserve.sh
kubectl get pods -n kserve
kubectl describe pods -n kserve
- name: Patch inferenceservice config
run: |
kubectl patch configmaps -n kserve inferenceservice-config --patch-file config/overlays/test/configmap/inferenceservice-ingress.yaml
kubectl describe configmaps -n kserve inferenceservice-config
- name: Run E2E tests with path-based routing
timeout-minutes: 50
run: |
./test/scripts/gh-actions/run-e2e-tests.sh "fast" "2"
- name: Check system status
if: always()
run: |
./test/scripts/gh-actions/status-check.sh
test-slow:
runs-on: ubuntu-latest
needs:
[kserve-image-build, predictor-runtime-build, explainer-runtime-build]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: "1.21"
- uses: actions/setup-python@v4
with:
python-version: "3.9"
- uses: ./.github/actions/free-up-disk-space
- uses: ./.github/actions/minikube-setup
- uses: ./.github/actions/kserve-dep-setup
- uses: ./.github/actions/base-download
- name: Download sklearn server image
uses: ishworkh/[email protected]
with:
image: "kserve/sklearnserver:${{ github.sha }}"
- name: Download xgb server image
uses: ishworkh/[email protected]
with:
image: "kserve/xgbserver:${{ github.sha }}"
- name: Download lgb server image
uses: ishworkh/[email protected]
with:
image: "kserve/lgbserver:${{ github.sha }}"
- name: Download paddle image
uses: ishworkh/[email protected]
with:
image: "kserve/paddleserver:${{ github.sha }}"
- name: Install Poetry and version plugin
run: ./test/scripts/gh-actions/setup-poetry.sh
- name: Install KServe
run: |
./test/scripts/gh-actions/setup-kserve.sh
kubectl get pods -n kserve
kubectl describe pods -n kserve
- name: Run E2E tests
timeout-minutes: 90
run: |
./test/scripts/gh-actions/run-e2e-tests.sh "slow" "2"
- name: Check system status
if: always()
run: |
./test/scripts/gh-actions/status-check.sh
test-explainer:
runs-on: ubuntu-latest
needs:
[kserve-image-build, predictor-runtime-build, explainer-runtime-build]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: "1.21"
- uses: actions/setup-python@v4
with:
python-version: "3.9"
- uses: ./.github/actions/free-up-disk-space
- uses: ./.github/actions/minikube-setup
- uses: ./.github/actions/kserve-dep-setup
- uses: ./.github/actions/base-download
- name: Download sklearn server image
uses: ishworkh/[email protected]
with:
image: "kserve/sklearnserver:${{ github.sha }}"
- name: Download alibi image
uses: ishworkh/[email protected]
with:
image: "kserve/alibi-explainer:${{ github.sha }}"
- name: Download art explainer image
uses: ishworkh/[email protected]
with:
image: "kserve/art-explainer:${{ github.sha }}"
- name: Install Poetry and version plugin
run: ./test/scripts/gh-actions/setup-poetry.sh
- name: Install KServe
run: |
./test/scripts/gh-actions/setup-kserve.sh
kubectl get pods -n kserve
kubectl describe pods -n kserve
- name: Run E2E tests
timeout-minutes: 40
run: |
./test/scripts/gh-actions/run-e2e-tests.sh "explainer" "2"
- name: Check system status
if: always()
run: |
./test/scripts/gh-actions/status-check.sh
test-transformer-mms:
runs-on: ubuntu-latest
needs:
[kserve-image-build, predictor-runtime-build, explainer-runtime-build]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: "1.21"
- uses: actions/setup-python@v4
with:
python-version: "3.9"
- uses: ./.github/actions/free-up-disk-space
- uses: ./.github/actions/minikube-setup
- uses: ./.github/actions/kserve-dep-setup
- uses: ./.github/actions/base-download
- name: Download sklearn server image
uses: ishworkh/[email protected]
with:
image: "kserve/sklearnserver:${{ github.sha }}"
- name: Download transformer image
uses: ishworkh/[email protected]
with:
image: "kserve/image-transformer:${{ github.sha }}"
- name: Download xgb server image
uses: ishworkh/[email protected]
with:
image: "kserve/xgbserver:${{ github.sha }}"
- name: Install Poetry and version plugin
run: ./test/scripts/gh-actions/setup-poetry.sh
- name: Install KServe
run: |
./test/scripts/gh-actions/setup-kserve.sh
kubectl get pods -n kserve
kubectl describe pods -n kserve
- name: Run E2E tests
timeout-minutes: 40
run: |
./test/scripts/gh-actions/run-e2e-tests.sh "transformer or mms or collocation" "2"
- name: Check system status
if: always()
run: |
./test/scripts/gh-actions/status-check.sh
test-qpext:
runs-on: ubuntu-latest
needs: [kserve-image-build, predictor-runtime-build]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: "1.21"
- uses: actions/setup-python@v4
with:
python-version: "3.9"
- uses: ./.github/actions/free-up-disk-space
- uses: ./.github/actions/minikube-setup
- uses: ./.github/actions/kserve-dep-setup
- uses: ./.github/actions/base-download
- name: Build queue proxy extension image
run: |
./test/scripts/gh-actions/build-qpext-image.sh
docker image ls
- name: Download sklearn server image
uses: ishworkh/[email protected]
with:
image: "kserve/sklearnserver:${{ github.sha }}"
- name: Install Poetry and version plugin
run: ./test/scripts/gh-actions/setup-poetry.sh
- name: Install KServe
run: |
./test/scripts/gh-actions/setup-kserve.sh
kubectl get pods -n kserve
kubectl describe pods -n kserve
- name: Patch qpext image
run: |
kubectl patch configmaps -n knative-serving config-deployment --patch '{"data": {"queue-sidecar-image": "kserve/qpext:${{ github.sha }}"}}'
kubectl describe configmaps -n knative-serving config-deployment
- name: Run queue proxy extension E2E tests
timeout-minutes: 30
run: |
./test/scripts/gh-actions/run-qpext-test.sh
- name: Check system status
if: always()
run: |
./test/scripts/gh-actions/status-check.sh
test-grpc:
runs-on: ubuntu-latest
needs:
[kserve-image-build, predictor-runtime-build, explainer-runtime-build]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: "1.21"
- uses: actions/setup-python@v4
with:
python-version: "3.9"
- uses: ./.github/actions/free-up-disk-space
- uses: ./.github/actions/minikube-setup
- uses: ./.github/actions/kserve-dep-setup
- uses: ./.github/actions/base-download
- name: Download custom model grpc image
uses: ishworkh/[email protected]
with:
image: "kserve/custom-model-grpc:${{ github.sha }}"
- name: Download custom transformer image
uses: ishworkh/[email protected]
with:
image: "kserve/image-transformer:${{ github.sha }}"
- name: Download custom transformer grpc image
uses: ishworkh/[email protected]
with:
image: "kserve/custom-image-transformer-grpc:${{ github.sha }}"
- name: Download xgb server image
uses: ishworkh/[email protected]
with:
image: "kserve/xgbserver:${{ github.sha }}"
- name: Download lgb server image
uses: ishworkh/[email protected]
with:
image: "kserve/lgbserver:${{ github.sha }}"
- name: Install Poetry and version plugin
run: ./test/scripts/gh-actions/setup-poetry.sh
- name: Install KServe
run: |
./test/scripts/gh-actions/setup-kserve.sh
kubectl get pods -n kserve
kubectl describe pods -n kserve
- name: Run E2E tests
timeout-minutes: 40
run: |
./test/scripts/gh-actions/run-e2e-tests.sh "grpc" "2"
- name: Check system status
if: always()
run: |
./test/scripts/gh-actions/status-check.sh
test-with-helm:
runs-on: ubuntu-latest
needs:
[kserve-image-build, predictor-runtime-build, explainer-runtime-build]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: "1.21"
- uses: actions/setup-python@v4
with:
python-version: "3.9"
- uses: ./.github/actions/free-up-disk-space
- uses: ./.github/actions/minikube-setup
- uses: ./.github/actions/kserve-dep-setup
- uses: ./.github/actions/base-download
- name: Install Poetry and version plugin
run: ./test/scripts/gh-actions/setup-poetry.sh
- name: Install Kserve from helm
run: |
./test/scripts/gh-actions/setup-modelmesh-dep.sh
./test/scripts/gh-actions/setup-kserve-helm.sh
kubectl get pods -n kserve
kubectl describe pods -n kserve
- name: Run E2E tests
timeout-minutes: 50
run: |
./test/scripts/gh-actions/run-e2e-tests.sh "helm"
kubectl get pods -n kserve
- name: Check system status
if: always()
run: |
./test/scripts/gh-actions/status-check.sh
test-raw:
runs-on: ubuntu-latest
needs:
[kserve-image-build, predictor-runtime-build, explainer-runtime-build]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
go-version: "1.21"
- uses: actions/setup-python@v4
with:
python-version: "3.9"
- uses: ./.github/actions/free-up-disk-space
- uses: ./.github/actions/minikube-setup
- uses: ./.github/actions/kserve-dep-setup
- uses: ./.github/actions/base-download
- name: Download sklearn server image
uses: ishworkh/[email protected]
with:
image: "kserve/sklearnserver:${{ github.sha }}"
- name: Download transformer image
uses: ishworkh/[email protected]
with:
image: "kserve/image-transformer:${{ github.sha }}"
- name: Install Poetry and version plugin
run: ./test/scripts/gh-actions/setup-poetry.sh
- name: Install KServe
run: |
./test/scripts/gh-actions/setup-kserve.sh
kubectl get pods -n kserve
kubectl describe pods -n kserve
- name: Run E2E tests
timeout-minutes: 40
run: |
./test/scripts/gh-actions/run-e2e-tests.sh "raw" "2"
- name: Check system status
if: always()
run: |
./test/scripts/gh-actions/status-check.sh
test-kourier:
runs-on: ubuntu-latest
needs:
[kserve-image-build, predictor-runtime-build, graph-tests-images-build]
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: "1.21"
- uses: actions/setup-python@v4
with:
python-version: "3.9"
- uses: ./.github/actions/free-up-disk-space
- uses: ./.github/actions/minikube-setup
- uses: ./.github/actions/kserve-dep-setup
with:
network-layer: "kourier"
- uses: ./.github/actions/base-download
- name: Download sklearn server image
uses: ishworkh/[email protected]
with:
image: "kserve/sklearnserver:${{ github.sha }}"
- name: Download xgb server image
uses: ishworkh/[email protected]
with:
image: "kserve/xgbserver:${{ github.sha }}"
- name: Download success_200_isvc predictor image
uses: ishworkh/[email protected]
with:
image: "kserve/success-200-isvc:${{ github.sha }}"
- name: Download error_404_isvc predictor image
uses: ishworkh/[email protected]
with:
image: "kserve/error-404-isvc:${{ github.sha }}"
- name: Install Poetry and version plugin
run: ./test/scripts/gh-actions/setup-poetry.sh
- name: Install KServe
run: |
./test/scripts/gh-actions/setup-kserve.sh
kubectl get pods -n kserve
kubectl describe pods -n kserve
- name: Patch inferenceservice config to disable istio
run: |
kubectl patch configmaps -n kserve inferenceservice-config --patch-file config/overlays/test/configmap/inferenceservice-disable-istio.yaml
kubectl describe configmaps -n kserve inferenceservice-config
- name: Run E2E tests
timeout-minutes: 120
run: |
export KSERVE_INGRESS_HOST_PORT=$(kubectl get pod -n knative-serving -l "app=3scale-kourier-gateway" \
--output=jsonpath="{.items[0].status.podIP}"):$(kubectl get pod -n knative-serving -l "app=3scale-kourier-gateway" \
--output=jsonpath="{.items[0].spec.containers[0].ports[0].containerPort}")
./test/scripts/gh-actions/run-e2e-tests.sh "kourier" "2"
kubectl get pods -n kserve
- name: Check system status
if: always()
run: |
./test/scripts/gh-actions/status-check.sh "kourier"