add How backups_to_keep_remote works #1332
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| pull_request: | |
| branches: | |
| - "*" | |
| push: | |
| branches: | |
| - master | |
| jobs: | |
| build: | |
| name: Build | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| golang-version: | |
| - "1.23" | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v4 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '^${{ matrix.golang-version }}' | |
| - name: Setup musl | |
| id: setup-musl | |
| run: | | |
| sudo apt-get install -y musl-tools musl-dev | |
| - name: Cache golang | |
| id: cache-golang | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/go/pkg/mod | |
| ~/.cache/go-build | |
| key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} | |
| - name: Install golang dependencies | |
| run: go mod download -x | |
| if: | | |
| steps.cache-golang.outputs.cache-hit != 'true' | |
| - name: Build clickhouse-backup binary | |
| id: make-race | |
| env: | |
| GOROOT: ${{ env.GOROOT_1_23_X64 }} | |
| run: | | |
| make build/linux/amd64/clickhouse-backup build/linux/arm64/clickhouse-backup | |
| make build/linux/amd64/clickhouse-backup-fips build/linux/arm64/clickhouse-backup-fips | |
| make build-race build-race-fips config test | |
| - name: Report unittest coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| file: _coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: unittest-${{ matrix.clickhouse }} | |
| # be careful with encrypt with old OpenSSL - https://habr.com/ru/post/535140/ | |
| # openssl enc -base64 -aes-256-cbc -e -in test/integration/credentials.json -out test/integration/credentials.json.enc -md md5 -k ${VAULT_PASSWORD} | |
| - name: Decrypting credentials for Google Cloud Storage | |
| id: secrets | |
| env: | |
| VAULT_PASSWORD: ${{ secrets.VAULT_PASSWORD }} | |
| run: | | |
| if [[ "" != "${VAULT_PASSWORD}" ]]; then | |
| openssl version | |
| openssl enc -base64 -aes-256-cbc -d -in test/integration/credentials.json.enc -out test/integration/credentials.json -md md5 -k ${VAULT_PASSWORD} | |
| fi | |
| echo "GCS_TESTS=$(if [ -z "${{ secrets.VAULT_PASSWORD }}" ]; then echo "false"; else echo "true"; fi)" >> $GITHUB_OUTPUT | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-gcp-credentials | |
| path: | | |
| test/integration/credentials.json | |
| if-no-files-found: error | |
| retention-days: 1 | |
| if: | | |
| steps.secrets.outputs.GCS_TESTS == 'true' | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-artifacts | |
| path: | | |
| build/linux/amd64/clickhouse-backup | |
| build/linux/arm64/clickhouse-backup | |
| build/linux/amd64/clickhouse-backup-fips | |
| build/linux/arm64/clickhouse-backup-fips | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-test-artifacts | |
| path: | | |
| clickhouse-backup/clickhouse-backup-race | |
| clickhouse-backup/clickhouse-backup-race-fips | |
| if-no-files-found: error | |
| retention-days: 1 | |
| outputs: | |
| GCS_TESTS: ${{ steps.secrets.outputs.GCS_TESTS }} | |
| testflows: | |
| needs: build | |
| name: Testflows | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| golang-version: | |
| - "1.23" | |
| clickhouse: | |
| - '22.3' | |
| - '22.8' | |
| - '23.3' | |
| - '23.8' | |
| - '24.3' | |
| - '24.8' | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v4 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '^${{ matrix.golang-version }}' | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: build-test-artifacts | |
| path: ./clickhouse-backup/ | |
| - name: Install python venv | |
| run: | | |
| set -x | |
| (dpkg -l | grep venv) || (apt-get update && apt-get install -y python3-venv) | |
| python3 -m venv ~/venv/qa | |
| - name: Cache python | |
| uses: actions/cache@v4 | |
| id: cache-python | |
| with: | |
| path: ~/venv/qa | |
| key: clickhouse-backup-python-${{ hashFiles('test/testflows/requirements.txt','.github/workflows/*.yaml') }} | |
| - name: Install python dependencies | |
| run: | | |
| set -x | |
| ~/venv/qa/bin/pip3 install -U -r ./test/testflows/requirements.txt | |
| if: | | |
| steps.cache-python.outputs.cache-hit != 'true' | |
| - name: Running TestFlows tests | |
| env: | |
| CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} | |
| QA_AWS_ACCESS_KEY: ${{ secrets.QA_AWS_ACCESS_KEY }} | |
| QA_AWS_ENDPOINT: ${{ secrets.QA_AWS_ENDPOINT }} | |
| QA_AWS_SECRET_KEY: ${{ secrets.QA_AWS_SECRET_KEY }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_BUCKET: ${{ secrets.QA_AWS_BUCKET }} | |
| QA_GCS_CRED_JSON: ${{ secrets.QA_GCS_CRED_JSON }} | |
| # don't change it to avoid not working CI/CD | |
| RUN_TESTS: "*" | |
| run: | | |
| set -xe | |
| export CLICKHOUSE_TESTS_DIR=$(pwd)/test/testflows/clickhouse_backup | |
| docker compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml pull | |
| chmod +x $(pwd)/clickhouse-backup/clickhouse-backup* | |
| source ~/venv/qa/bin/activate | |
| ~/venv/qa/bin/python3 ./test/testflows/clickhouse_backup/regression.py --debug --only="${RUN_TESTS:-*}" --log ./test/testflows/raw.log | |
| tfs --debug --no-colors transform compact ./test/testflows/raw.log ./test/testflows/compact.log | |
| tfs --debug --no-colors transform nice ./test/testflows/raw.log ./test/testflows/nice.log.txt | |
| tfs --debug --no-colors transform short ./test/testflows/raw.log ./test/testflows/short.log.txt | |
| tfs --debug --no-colors report results -a "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/" ./test/testflows/raw.log - --confidential --copyright "Altinity LTD" --logo ./test/testflows/altinity.png | ~/venv/qa/bin/tfs --debug --no-colors document convert > ./test/testflows/report.html | |
| sudo chmod -Rv +rx test/testflows/clickhouse_backup/_instances | |
| - name: Format testflows coverage | |
| env: | |
| GOROOT: ${{ env.GOROOT_1_23_X64 }} | |
| run: | | |
| sudo chmod -Rv a+rw test/testflows/_coverage_/ | |
| ls -la test/testflows/_coverage_ | |
| go env | |
| go tool covdata textfmt -i test/testflows/_coverage_/ -o test/testflows/_coverage_/coverage.out | |
| - name: Report testflows coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| base-path: ./ | |
| file: test/testflows/_coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: testflows-${{ matrix.clickhouse }} | |
| # todo possible failures https://github.com/actions/upload-artifact/issues/270 | |
| - name: Upload testflows logs | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: testflows-logs-and-reports-${{ matrix.clickhouse }}-${{ github.run_id }} | |
| path: | | |
| test/testflows/*.log | |
| test/testflows/*.log.txt | |
| test/testflows/clickhouse_backup/_instances/**/*.log | |
| test/testflows/*.html | |
| retention-days: 7 | |
| test: | |
| needs: build | |
| name: Test | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| golang-version: | |
| - "1.23" | |
| clickhouse: | |
| - '1.1.54394' | |
| - '19.17' | |
| - '20.3' | |
| - '20.8' | |
| - '21.3' | |
| - '21.8' | |
| - '22.3' | |
| - '22.8' | |
| - '23.3' | |
| - '23.8' | |
| - '24.3' | |
| - '24.8' | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v4 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '^${{ matrix.golang-version }}' | |
| # - name: Cache golang | |
| # id: cache-golang | |
| # uses: actions/cache@v4 | |
| # with: | |
| # path: | | |
| # ~/go/pkg/mod | |
| # ~/.cache/go-build | |
| # key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: build-test-artifacts | |
| path: ./clickhouse-backup/ | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: build-gcp-credentials | |
| path: ./test/integration/ | |
| if: | | |
| needs.build.outputs.GCS_TESTS == 'true' | |
| - name: Running integration tests | |
| env: | |
| RUN_PARALLEL: 3 | |
| GOROOT: ${{ env.GOROOT_1_23_X64 }} | |
| CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} | |
| # options for advanced debug CI/CD | |
| # RUN_TESTS: "TestLongListRemote" | |
| # LOG_LEVEL: "debug" | |
| # TEST_LOG_LEVEL: "debug" | |
| # GCS_DEBUG: "true" | |
| # SFTP_DEBUG: "true" | |
| # AZBLOB_DEBUG: "true" | |
| # FTP_DEBUG: "true" | |
| # S3_DEBUG: "true" | |
| CGO_ENABLED: 0 | |
| GCS_TESTS: ${{ needs.build.outputs.GCS_TESTS }} | |
| RUN_ADVANCED_TESTS: 1 | |
| AZURE_TESTS: 1 | |
| # need for FIPS | |
| QA_AWS_ACCESS_KEY: ${{ secrets.QA_AWS_ACCESS_KEY }} | |
| QA_AWS_SECRET_KEY: ${{ secrets.QA_AWS_SECRET_KEY }} | |
| QA_AWS_BUCKET: ${{ secrets.QA_AWS_BUCKET }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| # need for GCP over S3 | |
| QA_GCS_OVER_S3_ACCESS_KEY: ${{ secrets.QA_GCS_OVER_S3_ACCESS_KEY }} | |
| QA_GCS_OVER_S3_SECRET_KEY: ${{ secrets.QA_GCS_OVER_S3_SECRET_KEY }} | |
| QA_GCS_OVER_S3_BUCKET: ${{ secrets.QA_GCS_OVER_S3_BUCKET }} | |
| run: | | |
| set -xe | |
| echo "CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION}" | |
| echo "GCS_TESTS=${GCS_TESTS}" | |
| chmod +x $(pwd)/clickhouse-backup/clickhouse-backup* | |
| if [[ "${CLICKHOUSE_VERSION}" =~ 2[2-9]+ ]]; then | |
| export CLICKHOUSE_IMAGE=clickhouse/clickhouse-server | |
| else | |
| export CLICKHOUSE_IMAGE=yandex/clickhouse-server | |
| fi | |
| if [[ "${CLICKHOUSE_VERSION}" == 2* ]]; then | |
| export COMPOSE_FILE=docker-compose_advanced.yml | |
| else | |
| export COMPOSE_FILE=docker-compose.yml | |
| fi | |
| export CUR_DIR="$(pwd)/test/integration" | |
| export CLICKHOUSE_BACKUP_BIN="$(pwd)/clickhouse-backup/clickhouse-backup-race" | |
| docker compose -f "${CUR_DIR}/${COMPOSE_FILE}" --progress=quiet pull | |
| pids=() | |
| project_ids=() | |
| for ((i = 0; i < RUN_PARALLEL; i++)); do | |
| docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name project${i} --progress plain up -d & | |
| pids+=($!) | |
| project_ids+=("project${i}") | |
| done | |
| for index in "${!pids[@]}"; do | |
| pid=${pids[index]} | |
| project_id=${project_ids[index]} | |
| if wait "$pid"; then | |
| echo "$pid docker compose up successful" | |
| else | |
| echo "=== docker ${project_id} state ===" | |
| docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain ps -a | |
| echo "=== docker ${project_id} logs ===" | |
| docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain logs | |
| echo "$pid the docker compose up failed." | |
| exit 1 # Exit with an error code if any command fails | |
| fi | |
| done | |
| set +e | |
| go test -parallel ${RUN_PARALLEL} -timeout 90m -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v test/integration/integration_test.go | |
| TEST_FAILED=$? | |
| set -e | |
| if [[ "0" != "${TEST_FAILED}" ]]; then | |
| for project_id in "${project_ids[@]}"; do | |
| echo "=== docker ${project_id} state ===" | |
| docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain ps -a | |
| echo "=== docker ${project_id} logs ===" | |
| docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain logs | |
| done | |
| exit 1 | |
| fi | |
| - name: Format integration coverage | |
| env: | |
| GOROOT: ${{ env.GOROOT_1_23_X64 }} | |
| run: | | |
| sudo chmod -Rv a+rw test/integration/_coverage_/ | |
| ls -la test/integration/_coverage_ | |
| go tool covdata textfmt -i test/integration/_coverage_/ -o test/integration/_coverage_/coverage.out | |
| - name: Report integration coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| base-path: ./ | |
| file: test/integration/_coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: integration-${{ matrix.clickhouse }} | |
| coverage: | |
| needs: | |
| - test | |
| - testflows | |
| name: coverage | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Coveralls Finished | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| base-path: ./ | |
| parallel-finished: true | |
| docker: | |
| needs: | |
| - test | |
| - testflows | |
| name: Docker | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: build-artifacts | |
| path: ./build/linux/ | |
| - name: Extract DOCKER_TAG version | |
| id: docker_tag | |
| run: | | |
| DOCKER_TAG=${GITHUB_REF##*/} | |
| export DOCKER_TAG=${DOCKER_TAG##*\\} | |
| echo "docker_tag=${DOCKER_TAG:-dev}" >> $GITHUB_OUTPUT | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Building docker image | |
| env: | |
| DOCKER_REPO: ${{ secrets.DOCKER_REPO }} | |
| DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_USER: ${{ secrets.DOCKER_USER }} | |
| DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} | |
| DOCKER_TAG: ${{ steps.docker_tag.outputs.docker_tag }} | |
| run: | | |
| if [[ "${DOCKER_TOKEN}" != "" ]]; then | |
| export DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker.io} | |
| echo ${DOCKER_TOKEN} | docker login -u ${DOCKER_USER} --password-stdin ${DOCKER_REGISTRY} | |
| export VERSION=$(git describe --always --tags --abbrev=0 2>/dev/null | sed 's/^v//;s/\([^-]*-g\)/c\1/;s/-/./g') | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_short --pull --push . | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG}-fips --build-arg VERSION=$VERSION --target=image_fips --pull --push . | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}-full:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_full --pull --push . | |
| fi | |
| cleanup: | |
| name: Cleanup | |
| needs: | |
| - docker | |
| - coverage | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: delete build-artifacts | |
| uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| name: build-artifacts | |
| failOnError: false | |
| - name: delete build-gcp-credentials | |
| uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| name: build-gcp-credentials | |
| failOnError: false |