Skip to content

Restructure E2E tests #3136

Restructure E2E tests

Restructure E2E tests #3136

Workflow file for this run

name: CI pipeline for Claudie
on:
workflow_dispatch:
# Triggers the workflow on push or pull request events but only for the master branch
pull_request:
branches: [master]
env:
ENV_FILE: .env
SERVICES: manager builder terraformer ansibler kube-eleven kuber claudie-operator autoscaler-adapter testing-framework
jobs:
#--------------------------------------------------------------------------------------------------
select-test-sets:
if: github.event.pull_request.draft == false
runs-on: self-hosted
steps:
- name: Get PR labels
id: pr-labels
uses: joerick/[email protected]
- uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Print PR labels
id: print-pr-labels
working-directory: ./manifests/testing-framework
run: |
sudo apt update && sudo apt install -y wget tar
wget -q https://github.com/mikefarah/yq/releases/download/v4.27.2/yq_linux_amd64.tar.gz -O - |\
tar xz && mv yq_linux_amd64 yq
test-sets=$(./yq '.secretGenerator[].name' kustomization.yaml)
for set in $test-sets; do
echo $set
done
if [ -n "$GITHUB_PR_LABEL_TEST-SET-AUTOSCALING" ]; then
echo "autoscaling test-set"
fi
rm -f yq
merge-branch:
if: github.event.pull_request.draft == false
runs-on: self-hosted
needs: select-test-sets
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Merge with master branch
uses: everlytic/[email protected]
with:
github_token: ${{ github.token }}
source_ref: master
target_branch: ${{ github.head_ref }}
#--------------------------------------------------------------------------------------------------
check-changes:
if: github.event.pull_request.draft == false
runs-on: self-hosted
needs: merge-branch
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Check the repository for changes
id: change
run: |
arr=()
SERVICES=( ${{ env.SERVICES }} )
git fetch origin master
changes=($(git diff --name-only origin/master))
for i in "${changes[@]}"
do
if [[ "${i}" =~ .*\.md.* ]]; then
echo "Skipping document "$i""
continue
elif [[ "${i}" =~ .*(internal|proto)/.* ]] || [[ "${i}" =~ go\.(mod|sum) ]]; then
arr=(${SERVICES[@]})
echo "All services need to be built"
break
elif [[ "${i}" =~ .*manifests/.* ]]; then
echo "E2E test will run"
echo "RUN_TESTS=true" >> $GITHUB_OUTPUT
else
for SERVICE in "${SERVICES[@]}"; do
if [[ "${i}" =~ .*services/${SERVICE}.* ]]; then
arr+=($SERVICE)
echo "Detected change in $SERVICE"
fi
done
fi
done
echo "ARRAY_OF_CHANGES=$(echo ${arr[@]})" >> $GITHUB_OUTPUT
outputs:
ARRAY_OF_CHANGES: ${{ steps.change.outputs.ARRAY_OF_CHANGES }}
RUN_TESTS: ${{ steps.change.outputs.RUN_TESTS }}
#--------------------------------------------------------------------------------------------------
gotest:
runs-on: self-hosted
needs: [merge-branch, check-changes]
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Install golang
uses: actions/setup-go@v5
with:
go-version: "1.23.1"
- name: Run Go tests
run: go test -short ./...
#--------------------------------------------------------------------------------------------------
golangci:
name: Run golangci-lint
runs-on: self-hosted
needs: [merge-branch, check-changes]
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Install golang
uses: actions/setup-go@v5
with:
go-version: "1.23.1"
- name: golangci-lint
uses: golangci/golangci-lint-action@v6
with:
# Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
# It's highly recommended installing a specific version of golangci-lint from
# https://github.com/golangci/golangci-lint/releases
version: v1.60.3
# Optional: working directory, useful for monorepos
# working-directory: somedir
# Optional: golangci-lint command line arguments.
# args: --issues-exit-code=0
# Optional: show only new issues if it's a pull request. The default value is `false`.
# only-new-issues: true
# Optional: if set to true then the action will use pre-installed Go.
# skip-go-installation: true
# Optional: if set to true then the action don't cache or restore ~/go/pkg.
# skip-pkg-cache: true
# Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
# skip-build-cache: true
#--------------------------------------------------------------------------------------------------
build-and-push:
runs-on: self-hosted
needs: [merge-branch, check-changes]
steps:
- uses: actions/checkout@v4
if: (needs.check-changes.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false)
with:
ref: ${{ github.head_ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Set short sha output
if: (needs.check-changes.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false)
run: echo "SHORT_GITHUB_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV
- name: Set up docker
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# Update autoscaler-adapter manifest in this steps as new kuber would need to contain manifest with the correct image tag
- name: Edit autoscaler-adapter image tag in the manifest
if: ${{ needs.check-changes.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
id: autoscaler-tag
run: |
NEW_SERVICES=( ${{ needs.check-changes.outputs.ARRAY_OF_CHANGES }} )
# Check if autoscaler-adapter is going to be built, if so, insert latest tag and add kuber to array if needed.
if [[ "${NEW_SERVICES[*]}" =~ "autoscaler-adapter" ]]; then
TAG=${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
echo "Setting new tag for autoscaler adapter to $TAG"
sed -i "s/image: ghcr.io\/berops\/claudie\/autoscaler-adapter/&:$TAG/" services/kuber/templates/cluster-autoscaler.goyaml
if [[ ! "${NEW_SERVICES[*]}" =~ "kuber" ]]; then
NEW_SERVICES+=(kuber)
fi
else
# Check if kuber is going to be built, if so, insert latest tag and add to build.
if [[ "${NEW_SERVICES[*]}" =~ "kuber" ]]; then
TAG=${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
echo "Setting new tag for autoscaler adapter to $TAG"
sed -i "s/image: ghcr.io\/berops\/claudie\/autoscaler-adapter/&:$TAG/" services/kuber/templates/cluster-autoscaler.goyaml
NEW_SERVICES+=(autoscaler-adapter)
fi
fi
echo "ARRAY_OF_CHANGES=$(echo ${NEW_SERVICES[@]})" >> $GITHUB_OUTPUT
# Build the new images that were changed by a recent commit - tag image latest as well for testing
- name: Build and push new images
if: (needs.check-changes.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false)
run: |
ARR=(${{ steps.autoscaler-tag.outputs.ARRAY_OF_CHANGES }})
for SERVICE in "${ARR[@]}"
do
echo "-----Building $SERVICE-----"
IMGTAG="ghcr.io/berops/claudie/$SERVICE:${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}"
DOCKER_BUILDKIT=1 docker build --tag $IMGTAG -f ./services/$SERVICE/Dockerfile .
docker push $IMGTAG
done
outputs:
ARRAY_OF_CHANGES: ${{ steps.autoscaler-tag.outputs.ARRAY_OF_CHANGES }}
#--------------------------------------------------------------------------------------------------
edit-kustomization:
runs-on: self-hosted
needs: [merge-branch, check-changes, build-and-push, gotest, golangci]
steps:
- uses: actions/checkout@v4
if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
with:
ref: ${{ github.head_ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Set short sha output
if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
run: echo "SHORT_GITHUB_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV
- name: install kustomize
if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
uses: imranismail/setup-kustomize@v2
with:
kustomize-version: 4.5.6
# Set the new claudie image tags in kustomization.yaml
- name: Edit claudie kustomization.yaml
if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
working-directory: ./manifests/claudie
run: |
NEW_SERVICES=( ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES }} )
for SERVICE in "${NEW_SERVICES[@]}"
do
if [ "${SERVICE}" != "testing-framework" ]; then
echo "Setting a new tag for a $SERVICE"
kustomize edit set image ghcr.io/berops/claudie/$SERVICE:${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
fi
done
cat kustomization.yaml
# Set the new testing-framework image tags in kustomization.yaml
- name: Edit testing-framework kustomization.yaml
if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
working-directory: ./manifests/testing-framework
run: |
NEW_SERVICES=( ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES }} )
for SERVICE in "${NEW_SERVICES[@]}"
do
if [ "$SERVICE" == "testing-framework" ]; then
echo "Setting a new tag for a $SERVICE"
kustomize edit set image ghcr.io/berops/claudie/$SERVICE:${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
fi
done
cat kustomization.yaml
# Auto commit the changes
- name: Commit new kustomization.yaml to feature branch
if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
working-directory: ./manifests
run: |
BRANCH_NAME=${{ github.head_ref }}
git config --global user.name 'CI/CD pipeline'
git config --global user.email 'CI/[email protected]'
git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
git add claudie/kustomization.yaml
git add testing-framework/kustomization.yaml
git commit -m "Auto commit - update kustomization.yaml"
git push
#--------------------------------------------------------------------------------------------------
deploy-and-monitor:
runs-on: self-hosted
needs: [merge-branch, build-and-push, edit-kustomization, check-changes]
permissions:
id-token: write
contents: read
if: ${{ (needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' || needs.check-changes.outputs.RUN_TESTS == 'true') && github.event.pull_request.draft == false }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- name: Install terraform
run: |
sudo apt-get update && sudo apt-get install -y wget
wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
sudo apt update && sudo apt install terraform -y
- name: Set short sha output
run: echo "SHORT_GITHUB_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV
- name: Install kubectl
uses: azure/setup-kubectl@v4
with:
version: latest
- name: Install kustomize
uses: imranismail/setup-kustomize@v2
with:
kustomize-version: 4.5.6
- name: Set e2e kubeconfig
uses: azure/k8s-set-context@v4
with:
kubeconfig: ${{ secrets.E2E_CLUSTER_KUBECONFIG }}
- name: Get PR labels
id: pr-labels
uses: joerick/[email protected]
# Deploy services to new namespace
- name: Deploy to new namespace
working-directory: ./manifests/claudie
run: |
#set log level to debug
sed -i 's/GOLANG_LOG=info/GOLANG_LOG=debug/g' .env
#check if auto clean up is enabled/disabled
if [ -n "$GITHUB_PR_LABEL_DISABLE_CLEAN_UP" ]; then
echo "AUTO_CLEAN_UP=FALSE" >> .env
else
echo "AUTO_CLEAN_UP=TRUE" >> .env
fi
sudo apt update && sudo apt install -y wget tar
wget -q https://github.com/mikefarah/yq/releases/download/v4.27.2/yq_linux_amd64.tar.gz -O - |\
tar xz && mv yq_linux_amd64 yq
NAME_HASH="claudie-operator-role-binding-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}" ./yq e 'select(di == 0) * (select(.kind == "ClusterRoleBinding") | .metadata.name = strenv(NAME_HASH))' ./cluster-rbac/clusterrolebinding.yaml -i
NAMESPACES="claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER},e2e-secrets" ./yq eval 'select(documentIndex == 0).spec.template.spec.containers.0.env += [{"name": "CLAUDIE_NAMESPACES", "value": strenv(NAMESPACES)}]' -i operator.yaml
kustomize edit set namespace claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
kustomize build | kubectl apply -f -
cat kustomization.yaml
# Check if everything is ready and running
- name: Monitor status of the new namespace
run: |
arr=( ${{ env.SERVICES }} )
echo "${arr[@]}"
for SERVICE in "${arr[@]}"
do
if [ "${SERVICE}" != "testing-framework" ] && [ "${SERVICE}" != "autoscaler-adapter" ]; then
kubectl wait deployment -l app.kubernetes.io/name=$SERVICE --for=condition=available --timeout=900s --namespace=claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
fi
done
kubectl get pods --namespace=claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
- name: Insert random test hostnames to loadbalancer test set
working-directory: ./manifests/testing-framework/test-sets
run: |
sudo apt update && sudo apt install -y wget tar
wget -q https://github.com/mikefarah/yq/releases/download/v4.27.2/yq_linux_amd64.tar.gz -O - |\
tar xz && mv yq_linux_amd64 yq
HOSTNAME=$(echo $RANDOM | md5sum | head -c 20; echo;) ./yq e '.spec.loadBalancers.clusters.[1].dns.hostname = strenv(HOSTNAME)' test-set2/1.yaml -i
HOSTNAME=$(echo $RANDOM | md5sum | head -c 20; echo;) ./yq e '.spec.loadBalancers.clusters.[0].dns.hostname = strenv(HOSTNAME)' test-set2/3.yaml -i
- name: Create test static nodes
working-directory: ./manifests/testing-framework/test-sets
run: |
sudo apt update && sudo apt install -y jq
echo "Saving terraform script and key"
echo "${{ secrets.CREATE_STATIC_NODES }}" | base64 -d > script.tf
echo "Running terraform init"
terraform init > /dev/null 2>&1
echo "Running terraform apply"
terraform apply -auto-approve > /dev/null 2>&1
echo "Running terraform output"
IPS=$(terraform output --json | jq -r '.endpoints.value | values[]')
IP_ARR=( $IPS )
# use 3 nodes for the explicit static nodes test-set.
for file in test-set5/*; do
if [ -f "$file" ]; then
filename=$(basename "$file")
if [[ $filename == "1.yaml" ]]; then
ENDPOINT=${IP_ARR[1]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i
ENDPOINT=${IP_ARR[2]} ./yq e '.spec.nodePools.static.[0].nodes.[1].endpoint = strenv(ENDPOINT)' $file -i
fi
if [[ $filename == "2.yaml" ]]; then
ENDPOINT=${IP_ARR[1]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i
ENDPOINT=${IP_ARR[0]} ./yq e '.spec.nodePools.static.[0].nodes.[1].endpoint = strenv(ENDPOINT)' $file -i
fi
if [[ $filename == "3.yaml" ]]; then
ENDPOINT=${IP_ARR[2]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i
fi
fi
done
# use 1 node for the rolling update test.
for file in rolling-update/*; do
if [ -f "$file" ]; then
filename=$(basename "$file")
if [[ $filename == "1.yaml" ]]; then
ENDPOINT=${IP_ARR[3]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i
fi
if [[ $filename == "2.yaml" ]]; then
ENDPOINT=${IP_ARR[3]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i
fi
fi
done
#Clean up
rm -f yq
rm -f yq.1
rm -f install-man-page.sh
- name: Start the E2E tests
working-directory: ./manifests
run: |
sudo apt update && sudo apt install -y wget tar
wget -q https://github.com/mikefarah/yq/releases/download/v4.27.2/yq_linux_amd64.tar.gz -O - |\
tar xz && mv yq_linux_amd64 yq
NAME_HASH="testing-framework-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}" ./yq e -i '(select(.kind == "ClusterRoleBinding").metadata.name = strenv(NAME_HASH))' ./testing-framework/testing-framework.yaml
kustomize edit set namespace claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
kustomize build . | kubectl apply -f -
rm -f yq
- name: Monitor E2E test
run: |
# Wait for completion as background process - capture PID
kubectl wait --for=condition=complete --timeout=25000s job/testing-framework -n claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} &
completion_pid=$!
# Wait for failure as background process - capture PID
kubectl wait --for=condition=failed --timeout=25000s job/testing-framework -n claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} && exit 1 &
failure_pid=$!
# capture exit code of the first subprocess to exit
wait -n $completion_pid $failure_pid
exit_code=$?
if (( $exit_code == 0 )); then
echo "Testing-framework successful"
else
echo "Testing-framework received an error"
fi
# Exit 0 on success, 1 on failure
exit $exit_code
- name: Delete temporary namespace
run: |
kubectl delete namespace claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
- name: Destroy test static nodes
if: always()
working-directory: ./manifests/testing-framework/test-sets
run: |
echo "Running terraform init"
terraform init > /dev/null 2>&1
echo "Running terraform destroy"
terraform destroy -auto-approve > /dev/null 2>&1