From bbbb0d509c69d8b3578cb1df6a25d918d4f2fca6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=BC=A0=E7=A5=96=E5=BB=BA?= <zhangzujian.7@gmail.com>
Date: Sat, 15 Jun 2024 09:29:47 +0800
Subject: [PATCH] ci: check pod crashes on installation/e2e failure (#4160)

Signed-off-by: zhangzujian <zhangzujian.7@gmail.com>
---
 .github/workflows/build-x86-image.yaml | 75 ++++++++++++++++++++------
 1 file changed, 59 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
index 5e93bd1e4d4..fffad91aa66 100644
--- a/.github/workflows/build-x86-image.yaml
+++ b/.github/workflows/build-x86-image.yaml
@@ -476,12 +476,14 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
+        id: install
         env:
           VERSION: ${{ env.DEBUG_TAG }}
           DEBUG_WRAPPER: valgrind
         run: make kind-install-${{ matrix.mode }}-${{ matrix.ip-family }}
 
       - name: Run E2E
+        id: e2e
         working-directory: ${{ env.E2E_DIR }}
         env:
           E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
@@ -503,6 +505,7 @@ jobs:
           path: k8s-conformance-e2e-${{ matrix.ip-family }}-${{ matrix.mode }}-ko-log.tar.gz
 
       - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
 
       - name: Check valgrind result
@@ -639,12 +642,14 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
+        id: install
         env:
           VERSION: ${{ env.DEBUG_TAG }}
           DEBUG_WRAPPER: valgrind
         run: make kind-install-${{ matrix.ip-family }}
 
       - name: Run E2E
+        id: e2e
         working-directory: ${{ env.E2E_DIR }}
         run: make k8s-netpol-e2e
 
@@ -662,6 +667,7 @@ jobs:
           path: k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
 
       - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
 
       - name: Check valgrind result
@@ -773,12 +779,14 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
+        id: install
         env:
           VERSION: ${{ env.DEBUG_TAG }}
           DEBUG_WRAPPER: valgrind
         run: make kind-install-${{ matrix.ip-family }}
 
       - name: Run E2E
+        id: e2e
         working-directory: ${{ env.E2E_DIR }}
         run: make cyclonus-netpol-e2e
 
@@ -796,6 +804,7 @@ jobs:
           path: cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
 
       - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
 
       - name: Check valgrind result
@@ -926,12 +935,14 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
+        id: install
         env:
           VERSION: ${{ env.DEBUG_TAG }}
           DEBUG_WRAPPER: valgrind
         run: make kind-install-${{ matrix.mode }}-${{ matrix.ip-family }}
 
       - name: Run E2E
+        id: e2e
         working-directory: ${{ env.E2E_DIR }}
         env:
           E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
@@ -956,6 +967,7 @@ jobs:
           path: kube-ovn-conformance-e2e-${{ matrix.mode }}-${{ matrix.ip-family }}-ko-log.tar.gz
 
       - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
 
       - name: Check valgrind result
@@ -1083,9 +1095,11 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
+        id: install
         run: make kind-install-ovn-ic-${{ matrix.ip-family }}
 
       - name: Run E2E
+        id: e2e
         working-directory: ${{ env.E2E_DIR }}
         env:
           E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
@@ -1111,6 +1125,7 @@ jobs:
           path: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
 
       - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
 
   multus-conformance-e2e:
@@ -1205,12 +1220,14 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
+        id: install
         run: make kind-install-${{ matrix.ip-family }}
 
       - name: Install Multus
         run: make kind-install-multus
 
       - name: Run E2E
+        id: e2e
         working-directory: ${{ env.E2E_DIR }}
         env:
           E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
@@ -1231,6 +1248,7 @@ jobs:
           path: multus-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
 
       - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
 
   chart-test:
@@ -1279,13 +1297,14 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
+        id: install
         env:
           ENABLE_SSL: "${{ matrix.ssl }}"
         run: make kind-install-chart
 
       - name: Check kube ovn pod restarts
-        run: |
-          make check-kube-ovn-pod-restarts
+        if: ${{ success() || (failure() && steps.install.conclusion == 'failure') }}
+        run: make check-kube-ovn-pod-restarts
 
       - name: Uninstall Kube-OVN
         run: make kind-uninstall-chart
@@ -1330,11 +1349,12 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
+        id: install
         run: make kind-install-underlay-logical-gateway-dual
 
       - name: Check kube ovn pod restarts
-        run: |
-          make check-kube-ovn-pod-restarts
+        if: ${{ success() || (failure() && steps.install.conclusion == 'failure') }}
+        run: make check-kube-ovn-pod-restarts
 
       - name: Cleanup
         run: sh -x dist/images/cleanup.sh
@@ -1379,13 +1399,14 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN without LoadBalancer
+        id: install
         env:
           ENABLE_LB: "false"
         run: make kind-install
 
       - name: Check kube ovn pod restarts
-        run: |
-          make check-kube-ovn-pod-restarts
+        if: ${{ success() || (failure() && steps.install.conclusion == 'failure') }}
+        run: make check-kube-ovn-pod-restarts
 
       - name: Cleanup
         run: sh -x dist/images/cleanup.sh
@@ -1430,13 +1451,14 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
+        id: install
         env:
           ENABLE_NP: "false"
         run: make kind-install
 
       - name: Check kube ovn pod restarts
-        run: |
-          make check-kube-ovn-pod-restarts
+        if: ${{ success() || (failure() && steps.install.conclusion == 'failure') }}
+        run: make check-kube-ovn-pod-restarts
 
       - name: Cleanup
         run: sh -x dist/images/cleanup.sh
@@ -1534,17 +1556,19 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Multus and Kube-OVN
+        id: install
         run: make kind-install-lb-svc
 
       - name: Run E2E
+        id: e2e
         working-directory: ${{ env.E2E_DIR }}
         env:
           E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
         run: make kube-ovn-lb-svc-conformance-e2e
 
       - name: Check kube ovn pod restarts
-        run: |
-          make check-kube-ovn-pod-restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
+        run: make check-kube-ovn-pod-restarts
 
   webhook-e2e:
     name: Webhook E2E
@@ -1631,10 +1655,12 @@ jobs:
           sudo cp -r /root/.kube/ ~/.kube/
           sudo chown -R $(id -un). ~/.kube/
 
-      - name: Install Kube-OVN
+      - name: Install Kube-OVN with webhook
+        id: install
         run: make kind-install-webhook
 
       - name: Run E2E
+        id: e2e
         working-directory: ${{ env.E2E_DIR }}
         env:
           E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
@@ -1654,6 +1680,7 @@ jobs:
           path: webhook-e2e-ko-log.tar.gz
 
       - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
 
   installation-compatibility-test:
@@ -1696,11 +1723,12 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
+        id: install
         run: make kind-install
 
       - name: Check kube ovn pod restarts
-        run: |
-          make check-kube-ovn-pod-restarts
+        if: ${{ success() || (failure() && steps.install.conclusion == 'failure') }}
+        run: make check-kube-ovn-pod-restarts
 
       - name: kubectl ko log
         if: failure()
@@ -1821,9 +1849,11 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN with Cilium chaining
+        id: install
         run: make kind-install-cilium-chaining-${{ matrix.mode }}-${{ matrix.ip-family }}
 
       - name: Run E2E
+        id: e2e
         working-directory: ${{ env.E2E_DIR }}
         env:
           E2E_CILIUM_CHAINING: "true"
@@ -1846,6 +1876,7 @@ jobs:
           path: cilium-chaining-e2e-ko-log.tar.gz
 
       - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
 
       - name: Cleanup
@@ -1949,6 +1980,7 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
+        id: install
         env:
           NET_STACK: "${{ matrix.ip-family }}"
           ENABLE_SSL: "${{ matrix.ssl }}"
@@ -1956,6 +1988,7 @@ jobs:
         run: make kind-install-chart
 
       - name: Run E2E
+        id: e2e
         working-directory: ${{ env.E2E_DIR }}
         env:
           E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
@@ -1978,6 +2011,7 @@ jobs:
           path: kube-ovn-ha-e2e-${{ matrix.ssl }}-${{ matrix.bind-local }}-${{ matrix.ip-family }}-ko-log.tar.gz
 
       - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
 
       - name: Cleanup
@@ -2049,10 +2083,12 @@ jobs:
           sudo cp -r /root/.kube/ ~/.kube/
           sudo chown -R $(id -un). ~/.kube/
 
-      - name: Install Kube-OVN
+      - name: Install Kube-OVN and Submariner
+        id: install
         run: make kind-install-ovn-submariner
 
       - name: Run E2E
+        id: e2e
         working-directory: ${{ env.E2E_DIR }}
         run: make kube-ovn-submariner-conformance-e2e
 
@@ -2070,6 +2106,7 @@ jobs:
           path: kube-ovn-submariner-conformance-e2e-ko-log.tar.gz
 
       - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
 
       - name: Cleanup
@@ -2168,9 +2205,11 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN with VPC NAT gateway enabled
+        id: install
         run: make kind-install-vpc-nat-gw
 
       - name: Run E2E
+        id: e2e
         working-directory: ${{ env.E2E_DIR }}
         env:
           E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
@@ -2190,6 +2229,7 @@ jobs:
           path: iptables-vpc-nat-gw-conformance-e2e-ko-log.tar.gz
 
       - name: Check kube ovn pod restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
 
   ovn-vpc-nat-gw-conformance-e2e:
@@ -2277,15 +2317,18 @@ jobs:
           sudo chown -R $(id -un). ~/.kube/
 
       - name: Install Kube-OVN
+        id: install
         run: make kind-install
 
       - name: Run Vip E2E
+        id: vip-e2e
         working-directory: ${{ env.E2E_DIR }}
         env:
           E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
         run: make vip-conformance-e2e
 
       - name: Run Ovn VPC NAT GW E2E
+        id: vpc-e2e
         working-directory: ${{ env.E2E_DIR }}
         env:
           E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
@@ -2305,8 +2348,8 @@ jobs:
           path: ovn-vpc-nat-gw-conformance-e2e-ko-log.tar.gz
 
       - name: Check kube ovn pod restarts
-        run: |
-          make check-kube-ovn-pod-restarts
+        if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.vip-e2e.conclusion == 'failure' || steps.vpc-e2e.conclusion == 'failure')) }}
+        run: make check-kube-ovn-pod-restarts
 
   push:
     name: Push Images