Quali advanced testing #183
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Quali advanced testing | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| openwifi_revision: | |
| required: true | |
| default: 'main' | |
| description: 'revision of the Open Wifi Helm chart' | |
| ap_models: | |
| required: true | |
| default: 'edgecore_eap102,cig_wf194c4' | |
| description: 'the AP models to test' | |
| ap_version: | |
| required: true | |
| default: 'next-latest' | |
| description: 'revision of firmware to flash on AP, <branch>-<commit>' | |
| marker_expression: | |
| required: true | |
| default: 'advance' | |
| description: 'Marker expression to select tests to execute' | |
| tests_release: | |
| required: false | |
| default: "" | |
| description: "Tests release branch to use (i.e. 'release/v2.8.0' or 'master'). If left empty, latest release branch is used" | |
| schedule: | |
| - cron: '30 20 * * *' | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| vars: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| openwifi_revision: ${{ steps.vars.outputs.openwifi}} | |
| ap_models: ${{ steps.vars.outputs.ap_models}} | |
| ap_version: ${{ steps.vars.outputs.ap_version}} | |
| marker_expression: ${{ steps.vars.outputs.marker_expression }} | |
| tests_release: ${{ steps.vars.outputs.tests_release }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: set variables | |
| id: vars | |
| run: | | |
| echo "openwifi=$(echo ${{ github.event.inputs.openwifi_revision || 'main' }})" >> $GITHUB_OUTPUT | |
| echo "ap_models=$(echo ${{ github.event.inputs.ap_models || 'edgecore_eap102,cig_wf194c4' }})" >> $GITHUB_OUTPUT | |
| echo "ap_version=$(echo ${{ github.event.inputs.ap_version || 'next-latest' }})" >> $GITHUB_OUTPUT | |
| echo "marker_expression=$(echo ${{ github.event.inputs.marker_expression || 'advance' }})" >> $GITHUB_OUTPUT | |
| LATEST_TESTS_RELEASE=$(git branch -r | grep 'release/v' | sed 's!\s*origin/!!' | tail -1) | |
| if [[ -z "${{ github.event.inputs.tests_release }}" ]]; then | |
| echo "Tests release was not passed, using branch $LATEST_TESTS_RELEASE" | |
| echo "tests_release=$LATEST_TESTS_RELEASE" >> $GITHUB_OUTPUT | |
| else | |
| echo "Tests release was passed - ${{ github.event.inputs.tests_release }}" | |
| echo "tests_release=${{ github.event.inputs.tests_release }}" >> $GITHUB_OUTPUT | |
| fi | |
| generate-matrix: | |
| name: generate AP model matrix | |
| runs-on: ubuntu-latest | |
| needs: vars | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: generate-matrix | |
| id: set-matrix | |
| run: | | |
| AP_MODELS="${{ needs.vars.outputs.ap_models }}" | |
| AP_MODELS=$(echo $AP_MODELS | sed "s/,/\",\"/g" | sed 's/^/[\"/g' | sed 's/$/\"]/g') | |
| AP_MODELS=$(echo "$AP_MODELS" | jq -c 'map({"ap_model":.})') | |
| cat >> $GITHUB_OUTPUT << EOF | |
| matrix={"include":${AP_MODELS}} | |
| EOF | |
| build: | |
| runs-on: ubuntu-latest | |
| needs: ["vars"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.vars.outputs.tests_release }} | |
| - name: build and push Docker image | |
| uses: ./.github/actions/build-and-push-docker | |
| with: | |
| registry: tip-tip-wlan-cloud-docker-repo.jfrog.io | |
| registry_user: ${{ secrets.DOCKER_USER_NAME }} | |
| registry_password: ${{ secrets.DOCKER_USER_PASSWORD }} | |
| test: | |
| runs-on: [ self-hosted, small ] | |
| needs: [ vars, generate-matrix, build ] | |
| timeout-minutes: 720 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson( needs.generate-matrix.outputs.matrix ) }} | |
| env: | |
| AWS_EKS_NAME: tip-wlan-main | |
| AWS_DEFAULT_OUTPUT: json | |
| AWS_DEFAULT_REGION: ap-south-1 | |
| AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CLIENT_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CLIENT_KEY }} | |
| CLOUDSHELL_URL: quali-cloudshell.lab.wlan.tip.build | |
| CLOUDSHELL_USER: ${{ secrets.CLOUDSHELL_USER}} | |
| CLOUDSHELL_PASSWORD: ${{ secrets.CLOUDSHELL_PASSWORD }} | |
| MARKER_EXPRESSION: ${{ needs.vars.outputs.marker_expression }} | |
| steps: | |
| - name: install aws CLI tool | |
| run: | | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
| unzip awscliv2.zip | |
| sudo ./aws/install | |
| - name: install kubectl | |
| run: | | |
| # TODO WIFI-7839 revert to using stable when issue is resolved on AWS CLI side | |
| curl -s -LO "https://dl.k8s.io/release/v1.27.6/bin/linux/amd64/kubectl" | |
| sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.8' | |
| - name: checkout testing repo | |
| uses: actions/checkout@v3 | |
| - name: install dependencies | |
| run: pip install -r .quali/requirements.txt | |
| - name: start reservation | |
| run: | | |
| python .quali/start_reservation.py --global-inputs '{"Chart Version":"${{ needs.vars.outputs.openwifi_revision }}","AP Model":"${{ matrix.ap_model }}"}' --reservation-duration 720 "Advance Lab" | |
| - name: set reservation outputs | |
| if: always() | |
| id: reservation | |
| run: | | |
| echo "identifier=$(cat ./reservation_id.txt)" >> $GITHUB_OUTPUT | |
| echo "namespace=$(cat ./reservation_id.txt | cut -d "-" -f 1)" >> $GITHUB_OUTPUT | |
| - name: get EKS access credentials | |
| run: aws eks update-kubeconfig --name ${{ env.AWS_EKS_NAME }} | |
| - name: prepare namespace | |
| id: namespace | |
| run: | | |
| NAMESPACE="testing-${{ github.run_id }}-$(echo ${{ matrix.ap_model }} | tr '[:upper:]' '[:lower:]' | tr '_' '-')" | |
| echo "name=${NAMESPACE}" >> $GITHUB_OUTPUT | |
| - name: prepare configuration | |
| run: | | |
| cat << EOF > lab_info.json | |
| ${{ secrets.LAB_CONFIGURATION_JSON }} | |
| EOF | |
| sed -i 's/basic/advanced/g' lab_info.json | |
| DYN_CONFIG="$(python .quali/get_configuration.py --json ${{ steps.reservation.outputs.identifier }})" | |
| jq ".CONFIGURATION.advanced=$DYN_CONFIG" lab_info.json > lab_info.json.tmp && mv lab_info.json.tmp lab_info.json | |
| - name: run tests | |
| uses: ./.github/actions/run-tests | |
| with: | |
| namespace: ${{ steps.namespace.outputs.name }} | |
| testbed: advanced | |
| marker_expression: "${{ needs.vars.outputs.marker_expression }}" | |
| configuration_file: "./lab_info.json" | |
| testing_docker_image: tip-tip-wlan-cloud-docker-repo.jfrog.io/cloud-sdk-nightly:${{ github.run_id }} | |
| additional_args: '-o firmware="${{ needs.vars.outputs.ap_version }}"' | |
| allure_results_artifact_name: allure-results-${{ matrix.ap_model }} | |
| dns_records_to_resolve: "sec-${{ steps.reservation.outputs.namespace }}.cicd.lab.wlan.tip.build gw-${{ steps.reservation.outputs.namespace }}.cicd.lab.wlan.tip.build fms-${{ steps.reservation.outputs.namespace }}.cicd.lab.wlan.tip.build" | |
| # necessary because if conditionals in composite actions are currently not respected | |
| - name: get tests logs | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| podname=$(kubectl get pods -n ${{ steps.namespace.outputs.name }} --no-headers -o custom-columns=":metadata.name" -l job-name=testing | sed "s/pod\///") | |
| kubectl logs --timestamps -n ${{ steps.namespace.outputs.name }} $podname || true | |
| - name: delete namespace | |
| if: always() | |
| continue-on-error: true | |
| run: kubectl delete ns --ignore-not-found=true --wait ${{ steps.namespace.outputs.name }} | |
| - name: show gw logs | |
| if: failure() | |
| run: kubectl -n openwifi-${{ steps.reservation.outputs.namespace }} logs deployment/owgw | |
| - name: show fms logs | |
| if: failure() | |
| run: kubectl -n openwifi-${{ steps.reservation.outputs.namespace }} logs deployment/owfms | |
| - name: show prov logs | |
| if: failure() | |
| run: kubectl -n openwifi-${{ steps.reservation.outputs.namespace }} logs deployment/owprov | |
| - name: show sec logs | |
| if: failure() | |
| run: kubectl -n openwifi-${{ steps.reservation.outputs.namespace }} logs deployment/owsec | |
| - name: show analytics logs | |
| if: failure() | |
| run: kubectl -n openwifi-${{ steps.reservation.outputs.namespace }} logs deployment/analytics | |
| - name: show subscription (userportal) logs | |
| if: failure() | |
| run: kubectl -n openwifi-${{ steps.reservation.outputs.namespace }} logs deployment/owsub | |
| - name: stop reservation | |
| if: always() | |
| run: python .quali/stop_reservation.py ${{ steps.reservation.outputs.identifier }} | |
| report: | |
| needs: [ vars, generate-matrix, test ] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson( needs.generate-matrix.outputs.matrix ) }} | |
| steps: | |
| - name: checkout testing repo | |
| uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: allure-results-${{ matrix.ap_model }} | |
| path: allure-results | |
| - name: download history of previous run | |
| continue-on-error: true | |
| run: | | |
| LAST_RUN_ID=$(aws s3api head-object --bucket openwifi-allure-reports --key advance/${{ matrix.ap_model }}/latest/index.html | jq -r .Metadata.latest) | |
| aws s3 cp --recursive s3://openwifi-allure-reports/advance/${{ matrix.ap_model }}/$LAST_RUN_ID/history history | |
| - name: generate Allure report | |
| uses: ./.github/actions/generate-allure-report | |
| with: | |
| results_path: ./allure-results | |
| history_path: ./history | |
| additional_metadata: | | |
| Ap.Model=${{ matrix.ap_model }} | |
| - name: upload Allure report as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: allure-report-${{ matrix.ap_model }} | |
| path: allure-report | |
| # doing this to be able to aggregate multiple reports together later on | |
| - name: copy results into report | |
| run: | | |
| cp -r allure-results allure-report/results | |
| - name: upload to S3 | |
| if: github.ref == 'refs/heads/master' && needs.vars.outputs.marker_expression == 'advance' | |
| uses: ./.github/actions/allure-report-to-s3 | |
| with: | |
| test_type: advance | |
| testbed: ${{ matrix.ap_model }} | |
| report_path: allure-report | |
| s3_access_key_id: ${{ secrets.ALLURE_S3_ACCESS_KEY_ID }} | |
| s3_access_key_secret: ${{ secrets.ALLURE_S3_ACCESS_KEY_SECRET }} | |
| cleanup: | |
| needs: [ test ] | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: cleanup Docker image | |
| uses: ./.github/actions/cleanup-docker | |
| with: | |
| registry_user: ${{ secrets.DOCKER_USER_NAME }} | |
| registry_password: ${{ secrets.DOCKER_USER_PASSWORD }} |