CI-pipeline.yml

name: CI pipeline for Claudie
on:
  workflow_dispatch:
  # Triggers the workflow on push or pull request events but only for the master branch
  pull_request:
    branches: [master]

env:
  ENV_FILE: .env
  SERVICES: manager builder terraformer ansibler kube-eleven kuber claudie-operator autoscaler-adapter testing-framework

jobs:
  merge-branch:
    if: github.event.pull_request.draft == false
    runs-on: self-hosted
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: Merge with master branch
        uses: everlytic/branch-merge@1.1.5
        with:
          github_token: ${{ github.token }}
          source_ref: master
          target_branch: ${{ github.head_ref }}
  #--------------------------------------------------------------------------------------------------
  check-changes:
    if: github.event.pull_request.draft == false
    runs-on: self-hosted
    needs: merge-branch
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: Check the repository for changes
        id: change
        run: |
          arr=()
          SERVICES=( ${{ env.SERVICES }} )
          git fetch origin master
          changes=($(git diff  --name-only origin/master))
          for i in "${changes[@]}"
          do
            if [[ "${i}" =~ .*\.md.* ]]; then
              echo "Skipping document "$i""
              continue
            elif [[ "${i}" =~ .*(internal|proto)/.* ]] || [[ "${i}" =~ go\.(mod|sum) ]]; then
              arr=(${SERVICES[@]})
              echo "All services need to be built"
              break
            elif [[ "${i}" =~ .*manifests/.* ]]; then
              echo "E2E test will run"
              echo "RUN_TESTS=true" >> $GITHUB_OUTPUT
            else
              for SERVICE in "${SERVICES[@]}"; do
                if [[ "${i}" =~ .*services/${SERVICE}.* ]]; then
                arr+=($SERVICE)
                echo "Detected change in $SERVICE"
                fi
              done
            fi
          done
          echo "ARRAY_OF_CHANGES=$(echo ${arr[@]})" >> $GITHUB_OUTPUT
    outputs:
      ARRAY_OF_CHANGES: ${{ steps.change.outputs.ARRAY_OF_CHANGES }}
      RUN_TESTS: ${{ steps.change.outputs.RUN_TESTS }}
  #--------------------------------------------------------------------------------------------------
  select-test-sets:
    if: github.event.pull_request.draft == false
    runs-on: self-hosted
    needs: check-changes
    steps:
      - name: Get PR labels
        id: pr-labels
        uses: joerick/pr-labels-action@v1.0.9

      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: Print PR labels
        id: print-pr-labels
        working-directory: ./manifests/testing-framework
        run: |
          sudo apt update && sudo apt install -y wget tar
          wget -q https://github.com/mikefarah/yq/releases/download/v4.27.2/yq_linux_amd64.tar.gz -O - |\
          tar xz && mv yq_linux_amd64 yq

          #!/bin/bash

          test_sets=$(./yq '.secretGenerator[].name' kustomization.yaml)

          for set in $test_sets; do
            mnt="/go/services/testing-framework/test-sets/$set"

            if [ -n "$GITHUB_PR_LABEL_TEST-SET-AUTOSCALING" ] && [[ "$set" == *"autoscaling"* ]]; then
              ./yq -i "select(di ==0).spec.template.spec.volumes += [{\"name\": \"$set\", \"secret\": {\"secretName\": \"$set\"}}]"    testing-framework.yaml
              ./yq -i "select(di ==0).spec.template.spec.containers[].volumeMounts += [{\"name\": \"$set\", \"mountPath\": \"$mnt\"}]" testing-framework.yaml
            fi

            if [ -n "$GITHUB_PR_LABEL_TEST-SET-FAIL-UNTIL-LAST" ] && [[ "$set" == *"on-last"* ]]; then
              ./yq -i "select(di ==0).spec.template.spec.volumes += [{\"name\": \"$set\", \"secret\": {\"secretName\": \"$set\"}}]"    testing-framework.yaml
              ./yq -i "select(di ==0).spec.template.spec.containers[].volumeMounts += [{\"name\": \"$set\", \"mountPath\": \"$mnt\"}]" testing-framework.yaml
            fi
          done

          cat testing-framework.yaml

          rm -f yq
  #--------------------------------------------------------------------------------------------------
  gotest:
    runs-on: self-hosted
    needs: [merge-branch, check-changes, select-test-sets]
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: Install golang
        uses: actions/setup-go@v5
        with:
          go-version: "1.23.1"

      - name: Run Go tests
        run: go test -short ./...
  #--------------------------------------------------------------------------------------------------
  golangci:
    name: Run golangci-lint
    runs-on: self-hosted
    needs: [merge-branch, check-changes, select-test-sets]
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: Install golang
        uses: actions/setup-go@v5
        with:
          go-version: "1.23.1"

      - name: golangci-lint
        uses: golangci/golangci-lint-action@v6
        with:
          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
          # It's highly recommended installing a specific version of golangci-lint from
          # https://github.com/golangci/golangci-lint/releases
          version: v1.60.3

          # Optional: working directory, useful for monorepos
          # working-directory: somedir

          # Optional: golangci-lint command line arguments.
          # args: --issues-exit-code=0

          # Optional: show only new issues if it's a pull request. The default value is `false`.
          # only-new-issues: true

          # Optional: if set to true then the action will use pre-installed Go.
          # skip-go-installation: true

          # Optional: if set to true then the action don't cache or restore ~/go/pkg.
          # skip-pkg-cache: true

          # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
          # skip-build-cache: true

  #--------------------------------------------------------------------------------------------------
  build-and-push:
    runs-on: self-hosted
    needs: [merge-branch, check-changes, select-test-sets]
    steps:
      - uses: actions/checkout@v4
        if: (needs.check-changes.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false)
        with:
          ref: ${{ github.head_ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: Set short sha output
        if: (needs.check-changes.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false)
        run: echo "SHORT_GITHUB_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV

      - name: Set up docker
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      # Update autoscaler-adapter manifest in this steps as new kuber would need to contain manifest with the correct image tag
      - name: Edit autoscaler-adapter image tag in the manifest
        if: ${{ needs.check-changes.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
        id: autoscaler-tag
        run: |
          NEW_SERVICES=( ${{ needs.check-changes.outputs.ARRAY_OF_CHANGES }} )

          # Check if autoscaler-adapter is going to be built, if so, insert latest tag and add kuber to array if needed.
          if [[ "${NEW_SERVICES[*]}" =~ "autoscaler-adapter" ]]; then
            TAG=${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
            echo "Setting new tag for autoscaler adapter to $TAG"
            sed -i "s/image: ghcr.io\/berops\/claudie\/autoscaler-adapter/&:$TAG/" services/kuber/templates/cluster-autoscaler.goyaml

            if [[ ! "${NEW_SERVICES[*]}" =~ "kuber" ]]; then
              NEW_SERVICES+=(kuber)
            fi
          else
            # Check if kuber is going to be built, if so, insert latest tag and add to build.

            if [[ "${NEW_SERVICES[*]}" =~ "kuber" ]]; then
              TAG=${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
              echo "Setting new tag for autoscaler adapter to $TAG"
              sed -i "s/image: ghcr.io\/berops\/claudie\/autoscaler-adapter/&:$TAG/" services/kuber/templates/cluster-autoscaler.goyaml
              NEW_SERVICES+=(autoscaler-adapter)
            fi
          fi

          echo "ARRAY_OF_CHANGES=$(echo ${NEW_SERVICES[@]})" >> $GITHUB_OUTPUT

      # Build the new images that were changed by a recent commit - tag image latest as well for testing
      - name: Build and push new images
        if: (needs.check-changes.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false)
        run: |
          ARR=(${{ steps.autoscaler-tag.outputs.ARRAY_OF_CHANGES }})
          for SERVICE in "${ARR[@]}"
          do
            echo "-----Building $SERVICE-----"
            IMGTAG="ghcr.io/berops/claudie/$SERVICE:${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}" 
            DOCKER_BUILDKIT=1 docker build --tag $IMGTAG -f ./services/$SERVICE/Dockerfile . 
            docker push $IMGTAG
          done
    outputs:
      ARRAY_OF_CHANGES: ${{ steps.autoscaler-tag.outputs.ARRAY_OF_CHANGES }}
  #--------------------------------------------------------------------------------------------------
  edit-kustomization:
    runs-on: self-hosted
    needs: [merge-branch, check-changes, select-test-sets, build-and-push, golangci, gotest]
    steps:
      - uses: actions/checkout@v4
        if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
        with:
          ref: ${{ github.head_ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: Set short sha output
        if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
        run: echo "SHORT_GITHUB_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV

      - name: install kustomize
        if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
        uses: imranismail/setup-kustomize@v2
        with:
          kustomize-version: 4.5.6

      # Set the new claudie image tags in kustomization.yaml
      - name: Edit claudie kustomization.yaml
        if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
        working-directory: ./manifests/claudie
        run: |
          NEW_SERVICES=( ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES }} )
          for SERVICE in "${NEW_SERVICES[@]}"
          do
            if [ "${SERVICE}" != "testing-framework" ]; then
              echo "Setting a new tag for a $SERVICE"
              kustomize edit set image ghcr.io/berops/claudie/$SERVICE:${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
            fi
          done

          cat kustomization.yaml

      # Set the new testing-framework image tags in kustomization.yaml
      - name: Edit testing-framework kustomization.yaml
        if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
        working-directory: ./manifests/testing-framework
        run: |
          NEW_SERVICES=( ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES }} )
          for SERVICE in "${NEW_SERVICES[@]}"
          do
            if [ "$SERVICE" == "testing-framework" ]; then
              echo "Setting a new tag for a $SERVICE"
              kustomize edit set image ghcr.io/berops/claudie/$SERVICE:${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
            fi
          done

          cat kustomization.yaml

      # Auto commit the changes
      - name: Commit new kustomization.yaml to feature branch
        if: ${{ needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' && github.event.pull_request.draft == false }}
        working-directory: ./manifests
        run: |
          BRANCH_NAME=${{ github.head_ref }}
          git config --global user.name 'CI/CD pipeline'
          git config --global user.email 'CI/CD-pipeline@users.noreply.github.com'
          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
          git add claudie/kustomization.yaml
          git add testing-framework/kustomization.yaml
          git commit -m "Auto commit - update kustomization.yaml"
          git push

  #--------------------------------------------------------------------------------------------------
  deploy-and-monitor:
    runs-on: self-hosted
    needs: [merge-branch, build-and-push, edit-kustomization, check-changes, select-test-sets]
    permissions:
      id-token: write
      contents: read
    if: ${{ (needs.build-and-push.outputs.ARRAY_OF_CHANGES != '' || needs.check-changes.outputs.RUN_TESTS == 'true') && github.event.pull_request.draft == false }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: Install terraform
        run: | 
          sudo apt-get update && sudo apt-get install -y wget
          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
          echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
          sudo apt update && sudo apt install terraform -y

      - name: Set short sha output
        run: echo "SHORT_GITHUB_SHA=`echo ${GITHUB_SHA} | cut -c1-7`" >> $GITHUB_ENV

      - name: Install kubectl
        uses: azure/setup-kubectl@v4
        with:
          version: latest

      - name: Install kustomize
        uses: imranismail/setup-kustomize@v2
        with:
          kustomize-version: 4.5.6

      - name: Set e2e kubeconfig
        uses: azure/k8s-set-context@v4
        with:
          kubeconfig: ${{ secrets.E2E_CLUSTER_KUBECONFIG }}

      - name: Get PR labels
        id: pr-labels
        uses: joerick/pr-labels-action@v1.0.9

      # Deploy services to new namespace
      - name: Deploy to new namespace
        working-directory: ./manifests/claudie
        run: |
          #set log level to debug
          sed -i 's/GOLANG_LOG=info/GOLANG_LOG=debug/g' .env
          #check if auto clean up is enabled/disabled
          if [ -n "$GITHUB_PR_LABEL_DISABLE_CLEAN_UP" ]; then
            echo "AUTO_CLEAN_UP=FALSE" >> .env
          else
            echo "AUTO_CLEAN_UP=TRUE" >> .env
          fi

          sudo apt update && sudo apt install -y wget tar
          wget -q https://github.com/mikefarah/yq/releases/download/v4.27.2/yq_linux_amd64.tar.gz -O - |\
          tar xz && mv yq_linux_amd64 yq

          NAME_HASH="claudie-operator-role-binding-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}" ./yq e 'select(di == 0) * (select(.kind == "ClusterRoleBinding") | .metadata.name = strenv(NAME_HASH))' ./cluster-rbac/clusterrolebinding.yaml -i
          NAMESPACES="claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER},e2e-secrets" ./yq eval 'select(documentIndex == 0).spec.template.spec.containers.0.env += [{"name": "CLAUDIE_NAMESPACES", "value": strenv(NAMESPACES)}]' -i operator.yaml
          kustomize edit set namespace claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
          kustomize build | kubectl apply -f -

          cat kustomization.yaml

      # Check if everything is ready and running
      - name: Monitor status of the new namespace
        run: |
          arr=( ${{ env.SERVICES }} )
          echo "${arr[@]}"
          for SERVICE in "${arr[@]}"
            do
              if [ "${SERVICE}" != "testing-framework" ] && [ "${SERVICE}" != "autoscaler-adapter" ]; then
                kubectl wait deployment -l app.kubernetes.io/name=$SERVICE --for=condition=available --timeout=900s --namespace=claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
              fi
            done

          kubectl get pods --namespace=claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}

      - name: Insert random test hostnames to loadbalancer test set
        working-directory: ./manifests/testing-framework/test-sets
        run: |
          sudo apt update && sudo apt install -y wget tar
          wget -q https://github.com/mikefarah/yq/releases/download/v4.27.2/yq_linux_amd64.tar.gz -O - |\
          tar xz && mv yq_linux_amd64 yq

          HOSTNAME=$(echo $RANDOM | md5sum | head -c 20; echo;)  ./yq e '.spec.loadBalancers.clusters.[1].dns.hostname = strenv(HOSTNAME)' test-set2/1.yaml -i
          HOSTNAME=$(echo $RANDOM | md5sum | head -c 20; echo;)  ./yq e '.spec.loadBalancers.clusters.[0].dns.hostname = strenv(HOSTNAME)' test-set2/3.yaml -i

      - name: Create test static nodes
        working-directory: ./manifests/testing-framework/test-sets
        run: |
          sudo apt update && sudo apt install -y jq

          echo "Saving terraform script and key"
          echo "${{ secrets.CREATE_STATIC_NODES }}" | base64 -d > script.tf

          echo "Running terraform init"
          terraform init > /dev/null 2>&1
          echo "Running terraform apply"   
          terraform apply -auto-approve > /dev/null 2>&1

          echo "Running terraform output"   
          IPS=$(terraform output --json | jq -r '.endpoints.value | values[]')
          IP_ARR=( $IPS )
          
          # use 3 nodes for the explicit static nodes test-set.
          for file in test-set5/*; do
              if [ -f "$file" ]; then
                  filename=$(basename "$file")
                  if [[ $filename == "1.yaml" ]]; then
                      ENDPOINT=${IP_ARR[1]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i
                      ENDPOINT=${IP_ARR[2]} ./yq e '.spec.nodePools.static.[0].nodes.[1].endpoint = strenv(ENDPOINT)' $file -i
                  fi
                  if [[ $filename == "2.yaml" ]]; then
                      ENDPOINT=${IP_ARR[1]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i
                      ENDPOINT=${IP_ARR[0]} ./yq e '.spec.nodePools.static.[0].nodes.[1].endpoint = strenv(ENDPOINT)' $file -i
                  fi
                  if [[ $filename == "3.yaml" ]]; then
                      ENDPOINT=${IP_ARR[2]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i
                  fi
              fi
          done
          
          # use 1 node for the rolling update test.
          for file in rolling-update/*; do
              if [ -f "$file" ]; then
                  filename=$(basename "$file")
                  if [[ $filename == "1.yaml" ]]; then
                      ENDPOINT=${IP_ARR[3]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i
                  fi
                  if [[ $filename == "2.yaml" ]]; then
                      ENDPOINT=${IP_ARR[3]} ./yq e '.spec.nodePools.static.[0].nodes.[0].endpoint = strenv(ENDPOINT)' $file -i
                  fi
              fi
          done

          #Clean up
          rm -f yq 
          rm -f yq.1 
          rm -f install-man-page.sh

      - name: Start the E2E tests
        working-directory: ./manifests
        run: |
          sudo apt update && sudo apt install -y wget tar
          wget -q https://github.com/mikefarah/yq/releases/download/v4.27.2/yq_linux_amd64.tar.gz -O - |\
          tar xz && mv yq_linux_amd64 yq

          NAME_HASH="testing-framework-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}" ./yq e -i '(select(.kind == "ClusterRoleBinding").metadata.name = strenv(NAME_HASH))' ./testing-framework/testing-framework.yaml 
          
          cat ./testing-framework/testing-framework.yaml

          kustomize edit set namespace claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
          kustomize build . | kubectl apply -f -

          rm -f yq 

      - name: Monitor E2E test
        run: |
          # Wait for completion as background process - capture PID
          kubectl wait --for=condition=complete --timeout=25000s job/testing-framework -n claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} &
          completion_pid=$!

          # Wait for failure as background process - capture PID
          kubectl wait --for=condition=failed --timeout=25000s job/testing-framework -n claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER} && exit 1 &
          failure_pid=$! 

          # capture exit code of the first subprocess to exit
          wait -n $completion_pid $failure_pid
          exit_code=$?

          if (( $exit_code == 0 )); then
            echo "Testing-framework successful"
          else
            echo "Testing-framework received an error"
          fi
          # Exit 0 on success, 1 on failure
          exit $exit_code

      - name: Delete temporary namespace
        run: |
          kubectl delete namespace claudie-${SHORT_GITHUB_SHA}-${GITHUB_RUN_NUMBER}
      
      - name: Destroy test static nodes
        if: always()
        working-directory: ./manifests/testing-framework/test-sets
        run: |
          echo "Running terraform init"
          terraform init > /dev/null 2>&1
          echo "Running terraform destroy"
          terraform destroy -auto-approve > /dev/null 2>&1