helm(fix): correct mongo cleanup script #6946
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous | |
| on: | |
| push: | |
| branches: | |
| - "*" | |
| pull_request: | |
| branches: | |
| - "*" | |
| concurrency: | |
| group: ${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| duplication-check: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| inPR: ${{ steps.check.outputs.number != '' }} | |
| steps: | |
| - name: Check if push is in PR | |
| id: check | |
| uses: 8BitJonny/[email protected] | |
| with: | |
| filterOutClosed: true | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| build-generic: | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| runs-on: ubuntu-latest | |
| needs: duplication-check | |
| if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'push' && needs.duplication-check.outputs.inPR == 'false' ) }} | |
| strategy: | |
| matrix: | |
| app: [ web, node ] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v1 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v1 | |
| - id: auth | |
| name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github' | |
| service_account: '${{ secrets.DEV_GKE_SA }}' | |
| - name: Login to GCR | |
| uses: docker/login-action@v1 | |
| with: | |
| registry: gcr.io | |
| username: oauth2accesstoken | |
| password: '${{ steps.auth.outputs.access_token }}' | |
| - name: Login to GAR | |
| uses: docker/login-action@v1 | |
| with: | |
| registry: us-east1-docker.pkg.dev | |
| username: oauth2accesstoken | |
| password: '${{ steps.auth.outputs.access_token }}' | |
| - name: Get branch name | |
| id: branch-raw | |
| uses: tj-actions/[email protected] | |
| - name: Format branch name | |
| id: branch-name | |
| run: >- | |
| echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }} | |
| | awk '{print tolower($0)}' | |
| | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|' | |
| | sed 's/[^a-z0-9\.\-]//g') | |
| >> $GITHUB_OUTPUT | |
| - name: Get current date | |
| id: date | |
| run: echo "date=$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT | |
| - name: Generate image metadata | |
| id: meta | |
| uses: docker/metadata-action@v3 | |
| with: | |
| images: | | |
| gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }} | |
| us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }} | |
| # generate Docker tags based on the following events/attributes | |
| tags: | | |
| type=ref,event=branch | |
| type=sha,enable=true,priority=100,prefix=sha-,suffix=-${{ steps.date.outputs.date }},format=short | |
| type=sha | |
| flavor: | | |
| latest=true | |
| - name: build and push | |
| uses: docker/build-push-action@v3 | |
| with: | |
| # cache-from: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache | |
| # cache-to: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache, mode=max | |
| context: . | |
| push: true | |
| file: ./build/${{ matrix.app }}/Dockerfile | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-derived: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| needs: | |
| - duplication-check | |
| - build-generic | |
| strategy: | |
| matrix: | |
| app: [ asset, linker ] | |
| if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'push' && needs.duplication-check.outputs.inPR == 'false' ) }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v1 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v1 | |
| - id: auth | |
| name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github' | |
| service_account: '${{ secrets.DEV_GKE_SA }}' | |
| - name: Login to GCR | |
| uses: docker/login-action@v1 | |
| with: | |
| registry: gcr.io | |
| username: oauth2accesstoken | |
| password: '${{ steps.auth.outputs.access_token }}' | |
| - name: Login to GAR | |
| uses: docker/login-action@v1 | |
| with: | |
| registry: us-east1-docker.pkg.dev | |
| username: oauth2accesstoken | |
| password: '${{ steps.auth.outputs.access_token }}' | |
| - name: Get branch name | |
| id: branch-raw | |
| uses: tj-actions/[email protected] | |
| - name: Format branch name | |
| id: branch-name | |
| run: >- | |
| echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }} | |
| | awk '{print tolower($0)}' | |
| | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|' | |
| | sed 's/[^a-z0-9\.\-]//g') | |
| >> $GITHUB_OUTPUT | |
| - name: Get current date | |
| id: date | |
| run: echo "date=$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT | |
| - name: Generate image metadata | |
| id: meta | |
| uses: docker/metadata-action@v3 | |
| with: | |
| images: | | |
| gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }} | |
| us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }} | |
| # generate Docker tags based on the following events/attributes | |
| tags: | | |
| type=ref,event=branch | |
| type=sha,enable=true,priority=100,prefix=sha-,suffix=-${{ steps.date.outputs.date }},format=short | |
| type=sha | |
| flavor: | | |
| latest=true | |
| - name: Set outputs | |
| id: get-sha | |
| run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| - name: build and push | |
| uses: docker/build-push-action@v3 | |
| with: | |
| # cache-from: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache | |
| # cache-to: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache,mode=max | |
| context: . | |
| push: true | |
| build-args: | | |
| SRC_IMG=us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-web-${{ steps.branch-name.outputs.current_branch }}:sha-${{ steps.get-sha.outputs.sha_short }} | |
| file: ./build/${{ matrix.app }}/Dockerfile | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| ####### | |
| # Below Tests only run on PR | |
| ####### | |
| jest-tests: | |
| name: "Jest" # This name is referenced when slacking status | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name == 'pull_request' }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v2 | |
| - name: Install dependencies | |
| run: | | |
| npm install | |
| npm run build-prod | |
| - run: ls | |
| - run: pwd | |
| - name: Run Jest Tests | |
| run: npm run jest-gha | |
| - name: Handle Jest Test Results | |
| run: cat /home/runner/jestResults.json; STATUS=`jq ".numFailedTestSuites" /home/runner/jestResults.json`; exit $STATUS | |
| if: ${{ always() }} | |
| sandbox-deploy: | |
| if: ${{ github.event_name == 'pull_request' }} | |
| runs-on: ubuntu-latest | |
| needs: build-derived | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v2 | |
| - id: auth | |
| name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github' | |
| service_account: '${{ secrets.DEV_GKE_SA }}' | |
| - name: Setup GCloud | |
| uses: google-github-actions/setup-gcloud@v0 | |
| with: | |
| project_id: ${{ secrets.DEV_PROJECT }} | |
| install_components: 'gke-gcloud-auth-plugin' | |
| - name: Get branch name | |
| id: branch-raw | |
| uses: tj-actions/[email protected] | |
| - name: Format branch name | |
| id: branch-name | |
| run: >- | |
| echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }} | |
| | awk '{print tolower($0)}' | |
| | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|' | |
| | sed 's/[^a-z0-9\.\-]//g') | |
| >> $GITHUB_OUTPUT | |
| - name: Set outputs | |
| id: get-sha | |
| run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| - name: Set up yq | |
| uses: frenck/action-setup-yq@v1 | |
| # - name: setup helm | |
| # uses: azure/setup-helm@v3 | |
| - name: Authenticate GHA Runner To Target Cluster | |
| uses: google-github-actions/get-gke-credentials@v0 | |
| with: | |
| cluster_name: ${{secrets.DEV_GKE_CLUSTER}} | |
| location: ${{secrets.DEV_GKE_REGION}} | |
| project_id: ${{secrets.DEV_GCP_PROJECT}} | |
| - name: Deploy Sandbox | |
| run: ./build/ci/sandbox-helm-deploy.sh build/ci/values.yaml | |
| env: | |
| GIT_COMMIT: "${{ steps.get-sha.outputs.sha_short }}" | |
| BRANCH: "${{ steps.branch-name.outputs.current_branch }}" | |
| PROJECT_ID: "${{ secrets.DEV_PROJECT }}" | |
| NAMESPACE: "${{secrets.DEV_SANDBOX_NAMESPACE}}" | |
| sandbox-ready: | |
| if: ${{ github.event_name == 'pull_request' }} | |
| needs: sandbox-deploy | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Set outputs | |
| id: get-sha | |
| run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| - name: Wait for test sandbox to become available | |
| # https://gist.github.com/rgl/f90ff293d56dbb0a1e0f7e7e89a81f42 | |
| run: ./build/ci/waitForSandbox.bash | |
| env: | |
| WAIT_DURATION: "3000" | |
| GIT_COMMIT: "${{ steps.get-sha.outputs.sha_short }}" | |
| pytest-job: | |
| if: ${{ github.event_name == 'pull_request' }} | |
| name: "PyTest" # This name is referenced when slacking status | |
| needs: | |
| - sandbox-ready | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - id: auth | |
| name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github' | |
| service_account: '${{ secrets.DEV_GKE_SA }}' | |
| - name: Setup GCloud | |
| uses: google-github-actions/setup-gcloud@v0 | |
| with: | |
| project_id: ${{ secrets.DEV_PROJECT }} | |
| install_components: 'gke-gcloud-auth-plugin' | |
| - name: Authenticate GHA Runner To Target Cluster | |
| uses: google-github-actions/get-gke-credentials@v0 | |
| with: | |
| cluster_name: ${{secrets.DEV_GKE_CLUSTER}} | |
| location: ${{secrets.DEV_GKE_REGION}} | |
| project_id: ${{secrets.DEV_GCP_PROJECT}} | |
| - name: Set outputs | |
| id: get-sha | |
| run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| - name: Get branch name | |
| id: branch-raw | |
| uses: tj-actions/[email protected] | |
| - name: Format branch name | |
| id: branch-name | |
| run: >- | |
| echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }} | |
| | awk '{print tolower($0)}' | |
| | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|' | |
| | sed 's/[^a-z0-9\.\-]//g') | |
| >> $GITHUB_OUTPUT | |
| - name: Start Job | |
| run: envsubst '${GITHUB_RUN_ID},${DEPLOY_ENV},${WEB_IMAGE_NAME},${WEB_IMAGE_TAG},${TIMESTAMP}' < ./build/ci/pyTestPod.yaml | kubectl apply -f - | |
| env: | |
| # dependent on GITHUB_RUN_ID, which is implicitly passed in | |
| DEPLOY_ENV: sandbox-${{ steps.get-sha.outputs.sha_short }} | |
| WEB_IMAGE_NAME: us-east1-docker.pkg.dev/${{secrets.DEV_PROJECT}}/containers/sefaria-web-${{ steps.branch-name.outputs.current_branch }} | |
| WEB_IMAGE_TAG: sha-${{ steps.get-sha.outputs.sha_short }} | |
| - name: Wait For Job To Finish | |
| run: ./build/ci/waitForCIJob.bash | |
| timeout-minutes: 60 | |
| env: | |
| # dependent on GITHUB_RUN_ID, which is implicitly passed in | |
| TEST_NAME: pytest | |
| - name: Get Logs From Cluster and propogate test result | |
| run: "kubectl logs --tail=-1 -l ci-run=$GITHUB_RUN_ID,test-name=pytest; LASTLINE=`kubectl logs --tail=1 -l ci-run=$GITHUB_RUN_ID,test-name=pytest`; STAT=${LASTLINE: -1}; exit $STAT" | |
| - name: Cleanup pyTest Pod | |
| run: kubectl delete jobs -l ci-run=$GITHUB_RUN_ID,test-name=pytest | |
| if: always() | |
| ending-notification: | |
| runs-on: ubuntu-latest | |
| if: ${{ always() && github.event_name == 'pull_request' }} | |
| needs: | |
| - pytest-job | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-node@v2 | |
| with: | |
| node-version: '14' | |
| - run: npm ci | |
| working-directory: ./build/notify | |
| - run: node notifyEnd.js | |
| working-directory: ./build/notify | |
| env: | |
| SLACK_TEST_SUCCESS_WEBHOOK_URL: ${{secrets.SLACK_TEST_SUCCESS_WEBHOOK_URL}} | |
| SLACK_TEST_FAILURE_WEBHOOK_URL: ${{secrets.SLACK_TEST_FAILURE_WEBHOOK_URL}} | |
| GITUSER_SLACK_MAP: ${{secrets.GITUSER_SLACK_MAP}} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| sandbox-cleanup: | |
| name: "Clean up CI Sandbox" | |
| if: ${{ always() && github.event_name == 'pull_request' }} | |
| needs: | |
| - pytest-job | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - id: auth | |
| name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| token_format: 'access_token' | |
| workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github' | |
| service_account: '${{ secrets.DEV_GKE_SA }}' | |
| - name: Setup GCloud | |
| uses: google-github-actions/setup-gcloud@v0 | |
| with: | |
| project_id: ${{ secrets.DEV_PROJECT }} | |
| install_components: 'gke-gcloud-auth-plugin' | |
| - name: Authenticate GHA Runner To Target Cluster | |
| uses: google-github-actions/get-gke-credentials@v0 | |
| with: | |
| cluster_name: ${{secrets.DEV_GKE_CLUSTER}} | |
| location: ${{secrets.DEV_GKE_REGION}} | |
| project_id: ${{secrets.DEV_GCP_PROJECT}} | |
| - name: Set outputs | |
| id: get-sha | |
| run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT | |
| - name: check helm | |
| id: get-helm | |
| run: echo "count=$(helm list -n $NAMESPACE | grep $NAME | wc -l)" >> $GITHUB_OUTPUT | |
| env: | |
| NAMESPACE: ${{ secrets.DEV_SANDBOX_NAMESPACE }} | |
| NAME: sandbox-${{ steps.get-sha.outputs.sha_short }} | |
| - name: Uninstall | |
| run: helm delete sandbox-${{ steps.get-sha.outputs.sha_short }} -n ${{ secrets.DEV_SANDBOX_NAMESPACE }} --debug --timeout 10m0s | |
| if: steps.get-helm.outputs.count > 0 |