From 73f5aedc1acd1871828b66d9c92b983dc4b33845 Mon Sep 17 00:00:00 2001 From: Micah Nagel Date: Sat, 14 Dec 2024 16:08:09 -0700 Subject: [PATCH] chore: cleanup/fix log save --- .github/actions/debug-output/action.yaml | 25 -------------------- .github/actions/save-logs/action.yaml | 30 ++++++++++++++++++++++-- .github/workflows/test-aks.yaml | 9 +++++-- .github/workflows/test-eks.yaml | 9 +++++-- .github/workflows/test-rke2.yaml | 9 +++++-- 5 files changed, 49 insertions(+), 33 deletions(-) diff --git a/.github/actions/debug-output/action.yaml b/.github/actions/debug-output/action.yaml index dce852e85..334c3f7a6 100644 --- a/.github/actions/debug-output/action.yaml +++ b/.github/actions/debug-output/action.yaml @@ -4,12 +4,6 @@ name: debug-output description: "Print out basic debug info for a k8s cluster" -inputs: - suffix: - description: 'Suffix to append to the debug log' - required: false - default: '' - runs: using: composite steps: @@ -31,22 +25,3 @@ runs: uds zarf tools kubectl describe nodes | tee /tmp/debug-k-describe-node.log || true echo "::endgroup::" shell: bash - - name: Pepr Debug - run: | - echo "::group::Pepr Pod Status and Metrics" - uds zarf tools kubectl top pods -n pepr-system - uds zarf tools kubectl get pods -n pepr-system - echo "::endgroup::" - echo "::group::Fetch pepr logs" - uds zarf tools kubectl logs -n pepr-system -l app=pepr-uds-core > /tmp/pepr-logs.log - uds zarf tools kubectl logs -n pepr-system -l app=pepr-uds-core-watcher > /tmp/pepr-watcher-logs.log - echo "::endgroup::" - echo "::group::Describe Failed Packages" - FAILED_PACKAGES=($(uds zarf tools kubectl get package -A -o jsonpath="{range .items[?(@.status.phase!='Ready')]}{.metadata.name}{','}{.metadata.namespace}{'\n'}{end}")); for PACKAGE in "${FAILED_PACKAGES[@]}"; do PACKAGE_NAME=$(echo "$PACKAGE" | awk -F "," '{print $1}'); PACKAGE_NAMESPACE=$(echo "$PACKAGE" | awk -F "," '{print $2}'); uds zarf tools kubectl describe "$PACKAGE_NAME" -n "$PACKAGE_NAMESPACE"; echo; done - echo "::endgroup::" - shell: bash - - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 - with: - name: pepr-debug-logs${{ inputs.suffix }} - path: | - /tmp/pepr-*.log diff --git a/.github/actions/save-logs/action.yaml b/.github/actions/save-logs/action.yaml index 588c5437e..0a52f77bb 100644 --- a/.github/actions/save-logs/action.yaml +++ b/.github/actions/save-logs/action.yaml @@ -6,14 +6,19 @@ description: "Save debug logs" inputs: suffix: - description: 'Suffix to append to the debug log' + description: "Suffix to append to the debug log" required: false - default: '' + default: "" + distro: + description: "Kubernetes distribution used in this CI run" + required: false + default: "k3d" runs: using: composite steps: - name: Pull logs from containerd + if: ${{ inputs.distro == 'k3d' }} run: | CONTAINER_NAME="k3d-uds-server-0" if docker ps | grep -q "$CONTAINER_NAME"; then @@ -25,6 +30,7 @@ runs: shell: bash - name: Dump Node Logs + if: ${{ inputs.distro == 'k3d' }} run: | docker ps --filter "name=k3d" --format "{{.Names}}" | while read line; do docker logs "$line" 2> /tmp/$line.log @@ -38,14 +44,33 @@ runs: shell: bash - name: Move Playwright Artifacts + if: ${{ inputs.distro == 'k3d' }} # Currently only run on k3d run: | sudo mkdir -p /tmp/playwright sudo mv test/playwright/.playwright/* /tmp/playwright || true shell: bash + # Additional/specific debug for non-k3d clusters + - name: Pepr Debug + if: ${{ inputs.distro != 'k3d' }} + run: | + echo "::group::Pepr Pod Status and Metrics" + uds zarf tools kubectl top pods -n pepr-system + uds zarf tools kubectl get pods -n pepr-system + echo "::endgroup::" + echo "::group::Fetch pepr logs" + uds zarf tools kubectl logs -n pepr-system -l app=pepr-uds-core > /tmp/pepr-logs.log + uds zarf tools kubectl logs -n pepr-system -l app=pepr-uds-core-watcher > /tmp/pepr-watcher-logs.log + echo "::endgroup::" + echo "::group::Describe Failed Packages" + FAILED_PACKAGES=($(uds zarf tools kubectl get package -A -o jsonpath="{range .items[?(@.status.phase!='Ready')]}{.metadata.name}{','}{.metadata.namespace}{'\n'}{end}")); for PACKAGE in "${FAILED_PACKAGES[@]}"; do PACKAGE_NAME=$(echo "$PACKAGE" | awk -F "," '{print $1}'); PACKAGE_NAMESPACE=$(echo "$PACKAGE" | awk -F "," '{print $2}'); uds zarf tools kubectl describe "$PACKAGE_NAME" -n "$PACKAGE_NAMESPACE"; echo; done + echo "::endgroup::" + shell: bash + - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: debug-log${{ inputs.suffix }} + retention-days: 7 path: | /tmp/zarf-*.log /tmp/uds-*.log @@ -55,3 +80,4 @@ runs: /tmp/k3d-uds-*.log /tmp/playwright/output /tmp/playwright/reports + /tmp/pepr-*.log diff --git a/.github/workflows/test-aks.yaml b/.github/workflows/test-aks.yaml index 7341806c9..aea4f71d0 100644 --- a/.github/workflows/test-aks.yaml +++ b/.github/workflows/test-aks.yaml @@ -5,7 +5,7 @@ name: AKS Nightly Test on: schedule: - - cron: '0 6 * * *' # Runs at midnight Mountain every day + - cron: "0 6 * * *" # Runs at midnight Mountain every day pull_request: paths: - tasks/iac.yaml @@ -86,8 +86,13 @@ jobs: - name: Debug Output if: ${{ always() }} uses: ./.github/actions/debug-output + + - name: Save logs + if: always() + uses: ./.github/actions/save-logs with: - suffix: -${{ matrix.flavor }} + suffix: -aks-${{ matrix.flavor }} + distro: "aks" - name: Remove UDS Core if: always() diff --git a/.github/workflows/test-eks.yaml b/.github/workflows/test-eks.yaml index a280cbc89..b35ac88cf 100644 --- a/.github/workflows/test-eks.yaml +++ b/.github/workflows/test-eks.yaml @@ -5,7 +5,7 @@ name: EKS Nightly Test on: schedule: - - cron: '0 6 * * *' # Runs at midnight Mountain every day + - cron: "0 6 * * *" # Runs at midnight Mountain every day pull_request: paths: - tasks/iac.yaml @@ -91,8 +91,13 @@ jobs: - name: Debug Output if: ${{ always() }} uses: ./.github/actions/debug-output + + - name: Save logs + if: always() + uses: ./.github/actions/save-logs with: - suffix: -${{ matrix.flavor }} + suffix: -eks-${{ matrix.flavor }} + distro: "eks" - name: Remove UDS Core if: always() diff --git a/.github/workflows/test-rke2.yaml b/.github/workflows/test-rke2.yaml index 556357611..8846e68e4 100644 --- a/.github/workflows/test-rke2.yaml +++ b/.github/workflows/test-rke2.yaml @@ -5,7 +5,7 @@ name: RKE2 Nightly Test on: schedule: - - cron: '0 6 * * *' # Runs at midnight Mountain every day + - cron: "0 6 * * *" # Runs at midnight Mountain every day pull_request: paths: - tasks/iac.yaml @@ -95,8 +95,13 @@ jobs: - name: Debug Output if: ${{ always() }} uses: ./.github/actions/debug-output + + - name: Save logs + if: always() + uses: ./.github/actions/save-logs with: - suffix: -${{ matrix.flavor }} + suffix: -rke2-${{ matrix.flavor }} + distro: "rke2" - name: Remove UDS Core if: always()