diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7d14ef3c..a3de0915 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,14 +16,14 @@ jobs: strategy: matrix: golang-version: - - "1.21" + - "1.22" steps: - name: Checkout project - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup golang id: setup-go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: '^${{ matrix.golang-version }}' @@ -32,24 +32,24 @@ jobs: run: | sudo apt-get install -y musl-tools musl-dev - - name: Cache golang - id: cache-golang - uses: actions/cache@v3 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} +# - name: Cache golang +# id: cache-golang +# uses: actions/cache@v4 +# with: +# path: | +# ~/go/pkg/mod +# ~/.cache/go-build +# key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} - name: Install golang dependencies run: go mod download -x - if: | - steps.cache-golang.outputs.cache-hit != 'true' +# if: | +# steps.cache-golang.outputs.cache-hit != 'true' - name: Build clickhouse-backup binary id: make-race env: - GOROOT: ${{ env.GOROOT_1_20_X64 }} + GOROOT: ${{ env.GOROOT_1_22_X64 }} run: | make build/linux/amd64/clickhouse-backup build/linux/arm64/clickhouse-backup make build/linux/amd64/clickhouse-backup-fips build/linux/arm64/clickhouse-backup-fips @@ -76,7 +76,7 @@ jobs: fi echo "GCS_TESTS=$(if [ -z "${{ secrets.VAULT_PASSWORD }}" ]; then echo "false"; else echo "true"; fi)" >> $GITHUB_OUTPUT - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: build-gcp-credentials path: | @@ -86,7 +86,7 @@ jobs: if: | steps.secrets.outputs.GCS_TESTS == 'true' - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: build-artifacts path: | @@ -97,7 +97,7 @@ jobs: if-no-files-found: error retention-days: 1 - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: build-test-artifacts path: | @@ -114,15 +114,25 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: + golang-version: + - "1.22" clickhouse: - '22.3' - '22.8' - '23.3' + - '23.8' + - '24.3' steps: - name: Checkout project - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 + - name: Setup golang + id: setup-go + uses: actions/setup-go@v5 + with: + go-version: '^${{ matrix.golang-version }}' + + - uses: actions/download-artifact@v4 with: name: build-test-artifacts path: ./clickhouse-backup/ @@ -134,7 +144,7 @@ jobs: python3 -m venv ~/venv/qa - name: Cache python - uses: actions/cache@v3 + uses: actions/cache@v4 id: cache-python with: path: ~/venv/qa @@ -159,30 +169,26 @@ jobs: # don't change it to avoid not working CI/CD RUN_TESTS: "*" run: | - set -x + set -xe export CLICKHOUSE_TESTS_DIR=$(pwd)/test/testflows/clickhouse_backup - command -v docker-compose || (sudo apt-get update && sudo apt-get install -y python3-pip && pip3 install -U docker-compose) - docker-compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml pull + docker compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml pull chmod +x $(pwd)/clickhouse-backup/clickhouse-backup* source ~/venv/qa/bin/activate - set +e ~/venv/qa/bin/python3 ./test/testflows/clickhouse_backup/regression.py --debug --only="${RUN_TESTS:-*}" --log ./test/testflows/raw.log - if [[ "0" != "$?" ]]; then - docker-compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml logs clickhouse_backup - exit 1 - fi - set -e tfs --debug --no-colors transform compact ./test/testflows/raw.log ./test/testflows/compact.log tfs --debug --no-colors transform nice ./test/testflows/raw.log ./test/testflows/nice.log.txt tfs --debug --no-colors transform short ./test/testflows/raw.log ./test/testflows/short.log.txt tfs --debug --no-colors report results -a "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/" ./test/testflows/raw.log - --confidential --copyright "Altinity LTD" --logo ./test/testflows/altinity.png | ~/venv/qa/bin/tfs --debug --no-colors document convert > ./test/testflows/report.html sudo chmod -Rv +rx test/testflows/clickhouse_backup/_instances - name: Format testflows coverage + env: + GOROOT: ${{ env.GOROOT_1_22_X64 }} run: | sudo chmod -Rv a+rw test/testflows/_coverage_/ - ls -la test/testflows/_coverage_ + ls -la test/testflows/_coverage_ + go env go tool covdata textfmt -i test/testflows/_coverage_/ -o test/testflows/_coverage_/coverage.out - name: Report testflows coverage uses: coverallsapp/github-action@v2 @@ -191,17 +197,17 @@ jobs: parallel: true format: golang flag-name: testflows-${{ matrix.clickhouse }} -# todo wait when resolve https://github.com/actions/upload-artifact/issues/270 and uncomment -# - name: Upload testflows logs -# uses: actions/upload-artifact@v3 -# with: -# name: testflows-logs-and-reports-${{ matrix.clickhouse }}-${{ github.run_id }} -# path: | -# test/testflows/*.log -# test/testflows/*.log.txt -# test/testflows/clickhouse_backup/_instances/**/*.log -# test/testflows/*.html -# retention-days: 7 + # todo possible failures https://github.com/actions/upload-artifact/issues/270 + - name: Upload testflows logs + uses: actions/upload-artifact@v4 + with: + name: testflows-logs-and-reports-${{ matrix.clickhouse }}-${{ github.run_id }} + path: | + test/testflows/*.log + test/testflows/*.log.txt + test/testflows/clickhouse_backup/_instances/**/*.log + test/testflows/*.html + retention-days: 7 test: needs: build @@ -210,7 +216,7 @@ jobs: strategy: matrix: golang-version: - - "1.21" + - "1.22" clickhouse: - '1.1.54394' - '19.17' @@ -221,31 +227,33 @@ jobs: - '22.3' - '22.8' - '23.3' + - '23.8' + - '24.3' steps: - name: Checkout project - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup golang id: setup-go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: '^${{ matrix.golang-version }}' - - name: Cache golang - id: cache-golang - uses: actions/cache@v3 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} +# - name: Cache golang +# id: cache-golang +# uses: actions/cache@v4 +# with: +# path: | +# ~/go/pkg/mod +# ~/.cache/go-build +# key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: build-test-artifacts path: ./clickhouse-backup/ - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: build-gcp-credentials path: ./test/integration/ @@ -254,10 +262,16 @@ jobs: - name: Running integration tests env: + RUN_PARALLEL: 3 + GOROOT: ${{ env.GOROOT_1_22_X64 }} CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} - # don't change it to avoid broken CI/CD!!! - # RUN_TESTS: "TestFIPS" + # options for advanced debug CI/CD + # RUN_TESTS: "TestLongListRemote" # LOG_LEVEL: "debug" + # TEST_LOG_LEVEL: "debug" + # GCS_DEBUG: "true" + # SFTP_DEBUG: "true" + # AZBLOB_DEBUG: "true" # FTP_DEBUG: "true" # S3_DEBUG: "true" CGO_ENABLED: 0 @@ -274,7 +288,7 @@ jobs: QA_GCS_OVER_S3_SECRET_KEY: ${{ secrets.QA_GCS_OVER_S3_SECRET_KEY }} QA_GCS_OVER_S3_BUCKET: ${{ secrets.QA_GCS_OVER_S3_BUCKET }} run: | - set -x + set -xe echo "CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION}" echo "GCS_TESTS=${GCS_TESTS}" @@ -292,13 +306,30 @@ jobs: export COMPOSE_FILE=docker-compose.yml fi - command -v docker-compose || (apt-get update && apt-get install -y python3-pip && pip3 install -U docker-compose) - + export CUR_DIR="$(pwd)/test/integration" export CLICKHOUSE_BACKUP_BIN="$(pwd)/clickhouse-backup/clickhouse-backup-race" - docker-compose -f test/integration/${COMPOSE_FILE} up -d || ( docker-compose -f test/integration/${COMPOSE_FILE} ps -a && docker-compose -f test/integration/${COMPOSE_FILE} logs clickhouse && exit 1 ) - docker-compose -f test/integration/${COMPOSE_FILE} ps -a - go test -timeout 30m -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v test/integration/integration_test.go + docker compose -f "${CUR_DIR}/${COMPOSE_FILE}" --progress=quiet pull + + pids=() + for ((i = 0; i < RUN_PARALLEL; i++)); do + docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name project${i} --progress plain up -d & + pids+=($!) + done + + + for pid in "${pids[@]}"; do + if wait "$pid"; then + echo "$pid docker compose up successful" + else + echo "$pid the docker compose up failed. Exiting." + exit 1 # Exit with an error code if any command fails + fi + done + + go test -parallel ${RUN_PARALLEL} -timeout 60m -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v test/integration/integration_test.go - name: Format integration coverage + env: + GOROOT: ${{ env.GOROOT_1_22_X64 }} run: | sudo chmod -Rv a+rw test/integration/_coverage_/ ls -la test/integration/_coverage_ @@ -329,9 +360,9 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout project - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: build-artifacts path: ./build/linux/ @@ -344,9 +375,9 @@ jobs: echo "docker_tag=${DOCKER_TAG:-dev}" >> $GITHUB_OUTPUT - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Building docker image env: @@ -360,9 +391,11 @@ jobs: if [[ "${DOCKER_TOKEN}" != "" ]]; then export DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker.io} echo ${DOCKER_TOKEN} | docker login -u ${DOCKER_USER} --password-stdin ${DOCKER_REGISTRY} - docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG} --target=image_short --pull --push . - docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG}-fips --target=image_fips --pull --push . - docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}-full:${DOCKER_TAG} --target=image_full --pull --push . + export VERSION=$(git describe --always --tags --abbrev=0 2>/dev/null | sed 's/^v//;s/\([^-]*-g\)/c\1/;s/-/./g') + + docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_short --pull --push . + docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG}-fips --build-arg VERSION=$VERSION --target=image_fips --pull --push . + docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}-full:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_full --pull --push . fi cleanup: @@ -374,13 +407,13 @@ jobs: if: always() steps: - name: delete build-artifacts - uses: geekyeggo/delete-artifact@v2 + uses: geekyeggo/delete-artifact@v5 with: name: build-artifacts failOnError: false - name: delete build-gcp-credentials - uses: geekyeggo/delete-artifact@v2 + uses: geekyeggo/delete-artifact@v5 with: name: build-gcp-credentials failOnError: false diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e87a0364..54ea808f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -12,15 +12,15 @@ jobs: strategy: matrix: golang-version: - - "1.21" + - "1.22" steps: - name: Checkout project - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup golang id: setup-go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: '^${{ matrix.golang-version }}' @@ -30,19 +30,19 @@ jobs: sudo apt-get install --no-install-recommends -y make sudo gem install --no-document fpm - - name: Cache golang - id: cache-golang - uses: actions/cache@v3 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} +# - name: Cache golang +# id: cache-golang +# uses: actions/cache@v4 +# with: +# path: | +# ~/go/pkg/mod +# ~/.cache/go-build +# key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} - name: Install golang dependencies run: go mod download -x - if: | - steps.cache-golang.outputs.cache-hit != 'true' +# if: | +# steps.cache-golang.outputs.cache-hit != 'true' - name: Extract DOCKER_TAG version id: docker_tag @@ -55,7 +55,7 @@ jobs: - name: Prepare binaries id: make env: - GOROOT: ${{ env.GOROOT_1_20_X64 }} + GOROOT: ${{ env.GOROOT_1_22_X64 }} run: | make build build-fips config test #make build-fips-darwin @@ -97,7 +97,7 @@ jobs: - name: Create release id: create_release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 if: startsWith(github.ref, 'refs/tags/') with: tag_name: v${{ steps.docker_tag.outputs.docker_tag }} @@ -116,13 +116,13 @@ jobs: ${{ steps.make_packages.outputs.tgz_darwin_arm64 }} ${{ steps.make_packages.outputs.tgz_linux_amd64_fips }} ${{ steps.make_packages.outputs.tgz_linux_arm64_fips }} -# ${{ steps.make_packages.outputs.tgz_darwin_amd64_fips }} -# ${{ steps.make_packages.outputs.tgz_darwin_arm64_fips }} + # ${{ steps.make_packages.outputs.tgz_darwin_amd64_fips }} + # ${{ steps.make_packages.outputs.tgz_darwin_arm64_fips }} - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Building docker image env: @@ -135,7 +135,8 @@ jobs: run: | export DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker.io} echo ${DOCKER_TOKEN} | docker login -u ${DOCKER_USER} --password-stdin ${DOCKER_REGISTRY} + export VERSION=$(git describe --always --tags --abbrev=0 2>/dev/null | sed 's/^v//;s/\([^-]*-g\)/c\1/;s/-/./g') - docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG} --tag ${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:latest --target=image_short --pull --push . - docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG}-fips --tag ${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:latest-fips --target=image_fips --pull --push . - docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}-full:${DOCKER_TAG} --tag ${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}-full:latest --target=image_full --pull --push . + docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG} --tag ${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:latest --build-arg VERSION=$VERSION --target=image_short --pull --push . + docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG}-fips --tag ${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:latest-fips --build-arg VERSION=$VERSION --target=image_fips --pull --push . + docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}-full:${DOCKER_TAG} --tag ${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}-full:latest --build-arg VERSION=$VERSION --target=image_full --pull --push . diff --git a/.github/workflows/stable.yaml b/.github/workflows/stable.yaml new file mode 100644 index 00000000..98108dfe --- /dev/null +++ b/.github/workflows/stable.yaml @@ -0,0 +1,34 @@ +name: Make stable docker tag + +on: + workflow_dispatch: + inputs: + docker_tag: + description: 'Docker release tag' + required: true + +jobs: + docker-tag-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Pull, Tag, Push clickhouse-backup as stable + env: + DOCKER_REPO: ${{ secrets.DOCKER_REPO }} + DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + run: | + docker buildx imagetools create --progress plain --tag ${DOCKER_REGISTRY:-docker.io}/${DOCKER_REPO}/${DOCKER_IMAGE}:stable ${DOCKER_REGISTRY:-docker.io}/${DOCKER_REPO}/${DOCKER_IMAGE}:${{ github.event.inputs.docker_tag }} diff --git a/.gitignore b/.gitignore index 646104a8..5d635a69 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ build/ _instances/ _coverage_/ __pycache__/ -*.py[cod] \ No newline at end of file +*.py[cod] +vendor/ \ No newline at end of file diff --git a/ChangeLog.md b/ChangeLog.md index 64319f80..d65e591d 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,21 +1,386 @@ -# v2.4.0 (not released yet) +# v2.5.22 +IMPROVEMENTS +- redirect logs into stderr instead of stdout, fix [969](https://github.com/Altinity/clickhouse-backup/issues/969) + +# v2.5.21 +IMPROVEMENTS +- switch from `docker-compose` (python) to `docker compose` (golang) +- add parallel integration test execution fix [888](https://github.com/Altinity/clickhouse-backup/issues/888) + +BUG FIXES +- properly handle log_pointer=1 corner case for `check_replica_before_attach: true`, fix [967](https://github.com/Altinity/clickhouse-backup/issues/967) +- properly handle empty output for `list` command when `remote_storage: custom`, fix [963](https://github.com/Altinity/clickhouse-backup/issues/963), thanks @straysh +- fix corner cases when connect to S3 provider with self-signed TLS certificates, check `S3_DISABLE_CERT_VALIDATION=true` in tests fix [960](https://github.com/Altinity/clickhouse-backup/issues/960) + +# v2.5.20 +IMPROVEMENTS +- add `--restore-table-mapping` CLI and API parameter to `restore` and `restore_remote` command, fix [937](https://github.com/Altinity/clickhouse-backup/issues/937), thanks @nithin-vunet and @raspreet-vunet + +BUG FIXES +- remove trailing `/` from `object_disk_path` to properly `create` and `restore`, fix [946](https://github.com/Altinity/clickhouse-backup/issues/946) + +# v2.5.19 +BUG FIXES +- fix `restore --rbac` behavior when RBAC objects contains `-`, `.` or any special characters new fixes for [930](https://github.com/Altinity/clickhouse-backup/issues/930) + +# v2.5.18 +BUG FIXES +- add `clean` command to `POST /backup/actions` API handler, fix [945](https://github.com/Altinity/clickhouse-backup/issues/945) + +# v2.5.17 +BUG FIXES +- Fix wrong restoration of Materialized views with view name starting with digits for `--restore-table-mapping`, fix [942](https://github.com/Altinity/clickhouse-backup/pull/942), thanks @praveenthuwat + +# v2.5.16 +BUG FIXES +- allow backup/restore tables and databases which contains additional special characters set, fix [938](https://github.com/Altinity/clickhouse-backup/issues/938) +- properly restore environment variables to avoid failures in config.ValidateConfig in REST API mode, fix [940](https://github.com/Altinity/clickhouse-backup/issues/940) + +# v2.5.15 +IMPROVEMENTS +- increase `s3_request_timeout_ms` (23.7+) and turn off `s3_use_adaptive_timeouts` (23.11+) when `use_embedded_backup_restore: true` + +BUG FIXES +- fix hangs `create` and `restore` when CLICKHOUSE_MAX_CONNECTIONS=0, fix [933](https://github.com/Altinity/clickhouse-backup/issues/933) +- remove obsolete `CLICKHOUSE_EMBEDDED_BACKUP_THREADS`, `CLICKHOUSE_EMBEDDED_BACKUP_THREADS` these settings could configure only via server level, not profile and query settings after 23.3 + +# v2.5.14 +IMPROVEMENTS +- add http_send_timeout=300, http_receive_timeout=300 to embedded backup/restore operations +- explicitly set `allow_s3_native_copy` and `allow_azure_native_copy` settings when `use_embedding_backup_restore: true` + +BUG FIXES +- remove too aggressive logs for object disk upload and download operations during create and restore commands execution + +# v2.5.13 +IMPROVEMENTS +- return error instead of warning when replication in progress during restore operation + +BUG FIXES +- fixed wrong persistent behavior override for `--env` parameter when use it with API server, all 2.5.x versions was affected +- fixed errors during drop exists RBAC objects which contains special character, fix [930](https://github.com/Altinity/clickhouse-backup/issues/930) + +# v2.5.12 +IMPROVEMENTS +- added "object_disk_size" to upload and download command logs + +BUG FIXES +- fixed corner case in `API server` hang when `watch` background command failures, fix [929](https://github.com/Altinity/clickhouse-backup/pull/929) thanks @tadus21 +- removed requirement `compression: none` for `use_embedded_backup_restore: true` +- refactored to fix corner case for backup size calculation for Object disks and Embedded backup, set consistent algorithm for CLI and API `list` command behavior + +# v2.5.11 +BUG FIXES +- fixed another corner case for `restore --data=1 --env=CLICKHOUSE_SKIP_TABLE_ENGINES=liveview,WindowView` + +# v2.5.10 +BUG FIXES +- fixed corner case when `use_resumable_state: true` and trying download already present local backup don't return error backup already exists [926](https://github.com/Altinity/clickhouse-backup/issues/926) +- fixed another corner case for `restore --data=1 --env=CLICKHOUSE_SKIP_TABLE_ENGINES=dictionary,view` + + +# v2.5.9 +IMPROVEMENTS +- added to `--partitions` CLI and API parameter additional format `tablesPattern:partition1,partitionX` or `tablesPattern:(partition1),(partitionX)` fix https://github.com/Altinity/clickhouse-backup/issues/916 +- added system.backup_version and version into logs fix [917](https://github.com/Altinity/clickhouse-backup/issues/917) +- added progress=X/Y to logs fix [918](https://github.com/Altinity/clickhouse-backup/issues/918) + +BUG FIXES +- allow stopping api server when watch command is stopped, fix [922](https://github.com/Altinity/clickhouse-backup/pull/922), thanks @tadus21 +- fixed corner case for --env=CLICKHOUSE_SKIP_TABLE_ENGINES=dictionary,view + +# v2.5.8 +IMPROVEMENTS +- added OCI compliant labels to containers, thanks https://github.com/denisok +- increased default clickhouse queries timeout from `5m` to `30m` for allow freeze very large tables with object disks + +BUG FIXES +- fix corner cases for `ResumeOperationsAfterRestart` and `keep_backup_local: -1` behavior +- fix wrong file extension recognition during download for `access` and `configs` , fix https://github.com/Altinity/clickhouse-backup/issues/921 + +# v2.5.7 +BUG FIXES +- wrong skip tables by engine when empty variables value `CLICKHOUSE_SKIP_TABLE_ENGINES=engine,` instead of `CLICKHOUSE_SKIP_TABLE_ENGINES=engine` fix [915](https://github.com/Altinity/clickhouse-backup/issues/915) +- restore stop works, if RBAC objects present in backup but user which used for connect to clickhouse don't have RBAC GRANTS or `access_management`, 2.5.0+ affected, fix [914](https://github.com/Altinity/clickhouse-backup/issues/914) + +# v2.5.6 +BUG FIXES +- skip `ValidateObjectDiskConfig` for `--diff-from-remote` when object disk doesn't contains data fix [910](https://github.com/Altinity/clickhouse-backup/issues/910) + +# v2.5.5 +IMPROVEMENTS +- added `object_disk_server_side_copy_concurrency` with default value `32`, to avoid slow `create` or `restore` backup process which was restricted by `upload_concurrency` or `download_concurrency` options, fix [903](https://github.com/Altinity/clickhouse-backup/issues/903) + +BUG FIXES +- fixed `create --rbac` behavior when /var/lib/clickhouse/access not exists but present only `replicated` system.user_directories, fix [904](https://github.com/Altinity/clickhouse-backup/issues/904) + +# v2.5.4 +IMPROVEMENTS +- add `info` logging for `uploadObjectDiskParts` and `downloadObjectDiskParts` operation + +# v2.5.3 +BUG FIXES +- fixed `Unknown setting base_backup` for `use_embedded_backup_restore: true` and `create --diff-from-remote`, affected 2.5.0+ versions, fix [735](https://github.com/Altinity/clickhouse-backup/issues/735) + +# v2.5.2 +BUG FIXES +- fixed issue after [865](https://github.com/Altinity/clickhouse-backup/pull/865) we can't use `create_remote --diff-from-remote` for `remote_storage: custom`, affected versions 2.5.0, 2.5.1, fix [900](https://github.com/Altinity/clickhouse-backup/issue/900) + +# v2.5.1 +BUG FIXES +- fixed issue when set both `AWS_ROLE_ARN` and `S3_ASSUME_ROLE_ARN` then `S3_ASSUME_ROLE_ARN` shall have more priority than `AWS_ROLE_ARN` fix [898](https://github.com/Altinity/clickhouse-backup/issues/898) + +# v2.5.0 +IMPROVEMENTS +- complete removed support for legacy backups, created with version prior v1.0 +- removed `disable_progress_bar` config option and related progress bar code +- added `--delete-source` parameter for `upload` and `create_remote` commands to explicitly delete local backup during upload, fix [777](https://github.com/Altinity/clickhouse-backup/issues/777) +- added support for `--env ENV_NAME=value` cli parameter for allow dynamically overriding any config parameter, fix [821](https://github.com/Altinity/clickhouse-backup/issues/821) +- added support for `use_embedded_backup_restore: true` with empty `embedded_backup_disk` value, tested on S3/GCS over S3/AzureBlobStorage, fix [695](https://github.com/Altinity/clickhouse-backup/issues/695) +- `--rbac, --rbac-only, --configs, --configs-only` now works with `use_embedded_backup_restore: true` +- `--data` for `restore` with `use_embedded_backup_restore: true` will use `allow_non_empty_tables=true`, fix [756](https://github.com/Altinity/clickhouse-backup/issues/756) +- added `--diff-from-remote` parameter for `create` command, will copy only new data parts object disk data, also allows downloading properly object disk data from required backup during `restore`, fix [865](https://github.com/Altinity/clickhouse-backup/issues/865) +- added support of native Clickhouse incremental backup for `use_embedded_backup_restore: true` fix [735](https://github.com/Altinity/clickhouse-backup/issues/735) +- added `GCS_CHUNK_SIZE` config parameter, try to speed up GCS upload fix [874](https://github.com/Altinity/clickhouse-backup/pull/874), thanks @dermasmid +- added `--remote-backup` cli parameter to `tables` command and `GET /backup/table`, fix [778](https://github.com/Altinity/clickhouse-backup/issues/778) +- added `rbac_always_backup: true` option to default config, will create backup for RBAC objects automatically, restore still require `--rbac` to avoid destructive actions, fix [793](https://github.com/Altinity/clickhouse-backup/issues/793) +- added `rbac_conflict_resolution: recreate` option for RBAC object name conflicts during restore, fix [851](https://github.com/Altinity/clickhouse-backup/issues/851) +- added `upload_max_bytes_per_seconds` and `download_max_bytes_per_seconds` config options to allow throttling without CAP_SYS_NICE, fix [817](https://github.com/Altinity/clickhouse-backup/issues/817) +- added `clickhouse_backup_in_progress_commands` metric, fix [836](https://github.com/Altinity/clickhouse-backup/issues/836) +- switched to golang 1.22 +- updated all third-party SDK to latest versions +- added `clickhouse/clickhouse-server:24.3` to CI/CD + +BUG FIXES +- continue `S3_MAX_PARTS_COUNT` default value from `2000` to `4000` to continue decrease memory usage for S3 +- changed minimal part size for multipart upload in CopyObject from `5Mb` to `10Mb` +- restore SQL UDF functions after restore tables +- execute `ALTER TABLE ... DROP PARTITION` instead of `DROP TABLE` for `restore` and `restore_remote` with parameters `--data --partitions=...`, fix [756](https://github.com/Altinity/clickhouse-backup/issues/756) +- fix wrong behavior for `freeze_by_part` + `freeze_by_part_where`, fix [855](https://github.com/Altinity/clickhouse-backup/issues/855) +- apply `CLICKHOUSE_SKIP_TABLES_ENGINES` during `create` command +- fixed behavior for upload / download when .inner. table missing for `MATERIALIZED VIEW` by table pattern, fix [765](https://github.com/Altinity/clickhouse-backup/issues/765) +- fixed `ObjectDisks` + `CLICKHOUSE_USE_EMBEDDED_BACKUP_RESTORE: true` - shall skip upload object disk content, fix [799](https://github.com/Altinity/clickhouse-backup/issues/799) +- fixed connection to clickhouse-server behavior when long clickhouse-server startup time and `docker-entrypoint.d` processing, will infinitely reconnect each 5 seconds, until success, fix [857](https://github.com/Altinity/clickhouse-backup/issues/857) +- fixed `USE_EMBEDDED_BACKUP_RESTORE=true` behavior to allow using backup disk with type `local`, fix [882](https://github.com/Altinity/clickhouse-backup/issues/882) +- fixed wrong list command behavior, it shall scann all `system.disks` path not only default disk to find partially created backups, fix [873](https://github.com/Altinity/clickhouse-backup/issues/873) +- fixed create `--rbac` behavior, don't create access folder if no RBAC objects are present +- fixed behavior when `system.disks` contains disk which not present in any `storage_policies`, fix [845](https://github.com/Altinity/clickhouse-backup/issues/845) + +# v2.4.35 +IMPROVEMENTS +- set part size for `s3:CopyObject` minimum 128Mb, look details https://repost.aws/questions/QUtW2_XaALTK63wv9XLSywiQ/s3-sync-command-is-slow-to-start-on-some-data + +# v2.4.34 +BUG FIXES +- fixed wrong behavior for CLICKHOUSE_SKIP_TABLES_ENGINES for engine=EngineName without parameters + +# v2.4.33 +BUG FIXES +- fixed wrong anonymous authorization for serviceAccount in GCS, added `GCS_SKIP_CREDENTIALS` fix [848](https://github.com/Altinity/clickhouse-backup/issues/848), fix [847](https://github.com/Altinity/clickhouse-backup/pull/847), thanks @sanadhis + +# v2.4.32 +IMPROVEMENTS +- added ability to make custom endpoint for `GCS`, fix [837](https://github.com/Altinity/clickhouse-backup/pull/837), thanks @sanadhis + +BUG FIXES +- fixed wrong config validation for `object_disk_path` even when no object disk present in backup during `restore`, fix [842](https://github.com/Altinity/clickhouse-backup/issues/842) + +# v2.4.31 +IMPROVEMENTS +- added `check_sum_algorithm` parameter for `s3` config section with "" default value, to avoid useless CPU usage during upload to `S3` storage, additional fix [829](https://github.com/Altinity/clickhouse-backup/issues/829) +- `upload` will delete local backup if upload successful, fix [834](https://github.com/Altinity/clickhouse-backup/issues/834) + +BUG FIXES +- fixed miss checksum for CopyObject in `s3`, fix [835](https://github.com/Altinity/clickhouse-backup/issues/835), affected 2.4.30 +- fixed wrong behavior for `restore --rbac-only` and `restore --rbac` for backups which not contains any schema, fix [832](https://github.com/Altinity/clickhouse-backup/issues/832) + +# v2.4.30 +BUG FIXES +- fixed `download` command corner cases for increment backup for tables with projections, fix [830](https://github.com/Altinity/clickhouse-backup/issues/830) +- added more informative error during try to `restore` not exists local backup +- fixed `upload` command for S3 when object lock policy turned on, fix [829](https://github.com/Altinity/clickhouse-backup/issues/829) + +# v2.4.29 +IMPROVEMENTS +- added `AZBLOB_DEBUG` environment and `debug` config parameter in `azblob` section + +BUG FIXES +- force set `RefCount` to 0 during `restore` for parts in S3/GCS over S3/Azure disks, for properly works DROP TABLE / DROP DATABASE +- use `os.Link` instead `os.Rename` for ClickHouse 21.4+, to properly create backup object disks +- ignore `frozen_metadata` during, create, upload, download and restore commands, fix [826](https://github.com/Altinity/clickhouse-backup/issues/826) +- `allow_parallel: true` doesn't work after execute list command, fix [827](https://github.com/Altinity/clickhouse-backup/issues/827) +- fixed corner cases, when disk has encrypted type and underlying disk is object storage + +# v2.4.28 +IMPROVEMENT +- refactoring `watch` command, after https://github.com/Altinity/clickhouse-backup/pull/804 +BUG FIXES +- fixed deletion for `object_disk_path` and `embedded` backups, after `upload` to properly respect `backups_to_keep_remote` + +# v2.4.27 +BUG FIXES +- fixed deletion for `object_disk_path` (all backups with S3, GCS over S3, AZBLOB disks from 2.4.0 to2.4.25 didn't properly delete their data from backup bucket) + +# v2.4.26 +IMPROVEMENTS +- improved re-balance disk during download if disk does not exist in `system.disks`. Use least used for `local` disks and `random` for object disks, fix [561](https://github.com/Altinity/clickhouse-backup/issues/561) + +BUG FIXES +- fixed regression `check_parts_columns` for Enum types (2.4.24+), fix [823](https://github.com/Altinity/clickhouse-backup/issues/823) +- properly applying marcos to `object_disk_path` during `delete` + +# v2.4.25 +BUG FIXES +- fixed `--restore-table-mapping` corner cases for when a destination database contains special characters, fix [820](https://github.com/Altinity/clickhouse-backup/issues/820) + +# v2.4.24 +BUG FIXES +- fixed `check_parts_columns` corner cases for `AggregateFunction` and `SimpleAggregateFunction` versioning, fix [819](https://github.com/Altinity/clickhouse-backup/issues/819) + +# v2.4.23 +IMPROVEMENTS +- refactored of `restore` command to allow parallel execution of `ALTER TABLE ... ATTACH PART` and improve parallelization of CopyObject during restore. + +# v2.4.22 +BUG FIXES +- changed `S3_MAX_PARTS_COUNT` default value from `256` to `2000` to fix memory usage for s3 which increased for 2.4.16+ + +# v2.4.21 +BUG FIXES +- refactored execution UpdateBackupMetrics, to avoid context canceled error, fix [814](https://github.com/Altinity/clickhouse-backup/issues/814) + +# v2.4.20 +IMPROVEMENTS +- refactored of `create` command to allow parallel execution of `FREEZE` and `UNFREEZE` and table level parallelization `object_disk.CopyObject` +- added `CLICKHOUSE_MAX_CONNECTIONS` config parameter to allow parallel execution `FREEZE` / `UNFREEZE` +- change `go.mod` to allow `GO111MODULE=on go install github.com/Altinity/clickhouse-backup/v2/cmd/clickhouse-backup@latest` + +# v2.4.19 +BUG FIXES +- use single `s3:CopyObject` call instead `s3:CreateMultipartUpload+s3:UploadCopyPart+s3:CompleteMultipartUpload` for files with size less 5Gb + +# v2.4.18 +BUG FIXES +- removed `HeadObject` request to calculate source key size in `CopyObject`, to allow cross region S3 disks backup, fix [813](https://github.com/Altinity/clickhouse-backup/issues/813) +- make optional `/backup/kill` query parameter `command` and optional arguments for `kill` command handled via `/backup/actions`, if omitted then will kill first command in "In progress" status, fix [808](https://github.com/Altinity/clickhouse-backup/issues/808) + +# v2.4.17 +BUG FIXES +- skip `CopyObject` execution for keys which have zero sizes, to allow properly backup `S3`, `GCS over S3` and `Azure` disks + +# v2.4.16 +BUG FIXES +- increased `AZBLOB_TIMEOUT` to 4h, instead 15m to allow downloading long size data parts +- changed `S3_MAX_PARTS_COUNT` default from `5000` to `256` and minimal `S3_PART_SIZE` from 5Mb to 25Mb from by default to improve speedup S3 uploading / downloading + +# v2.4.15 +BUG FIXES +- fixed `create` and `restore` command for ReplicatedMergeTree tables with `frozen_metadata.txt` parsing + +# v2.4.14 +IMPROVEMENTS +- refactored `semaphore.NewWeighted()` to `errgroup.SetLimit()` +- added parallelization to `create` and `restore` command during call `CopyObject` + +# v2.4.13 +BUG FIXES +- fixed `object_disk.CopyObject` during restore to allow use properly S3 endpoint +- fixed AWS IRSA environments handler, fix [798](https://github.com/Altinity/clickhouse-backup/issues/798) + +# v2.4.12 +BUG FIXES +- fixed `object_disk.CopyObject` to use simple `CopyObject` call, instead of multipart for zero object size, for backup S3 disks + +# v2.4.11 +BUG FIXES +- fixed `CopyObject` multipart upload complete Parts must be ordered by part number, for backup S3 disks + +# v2.4.10 +IMPROVEMENTS +- updated go modules to latest versions +- added `S3_REQUEST_PAYER` config parameter, look https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html for details, fix [795](https://github.com/Altinity/clickhouse-backup/issues/795) + +# v2.4.9 +BUG FIXES +- fixed list remote command date parsing for all `remote_storage: custom` integration examples +- `clickhouse-backup` should not fail when `--rbac` used but rbac object is not present in backup, but it should log warnings/errors, partial fix [793](https://github.com/Altinity/clickhouse-backup/issues/793) + +# v2.4.8 +BUG FIXES +- fixed Object Disks path parsing from config, remove unnecessary "/" +- if `S3_ACL` is empty then will not use ACL in PutObject, fix [785](https://github.com/Altinity/clickhouse-backup/issues/785) + +# v2.4.7 +BUG FIXES +- `--partitions=(v1,'v2')` could calculate wrong partition expression if `system.columns` will return fields in different order than they described in PARTITION BY clause, fix [791](https://github.com/Altinity/clickhouse-backup/issues/791) + +# v2.4.6 +IMPROVEMENTS +- make 'kopia' custom scripts really increment fix [781](https://github.com/Altinity/clickhouse-backup/issues/781) +- added `force_http` and improve retries in GCS upload [784](https://github.com/Altinity/clickhouse-backup/pull/784), thanks @minguyen9988 + +BUG FIXES +- added `Array(Tuple())` to exclude list for `check_parts_columns:true`, fix [789](https://github.com/Altinity/clickhouse-backup/issues/789) +- fixed `delete remote` command for s3 buckets with enabled versioning, fix [782](https://github.com/Altinity/clickhouse-backup/issues/782) +- fixed panic during create integration tables when `API_LISTEN` doesn't contain ":" character, fix [790](https://github.com/Altinity/clickhouse-backup/issues/790) + +# v2.4.5 +BUG FIXES +- added aws.LogResponse for `S3_DEBUG` (affected 2.4.4+ versions) + +# v2.4.4 +BUG FIXES +- removed `aws.LogResponseWithBody` for `S3_DEBUG` to avoid too many logs (affected 2.4.0+ versions) + +# v2.4.3 +IMPROVEMENTS +- added `list` command to API /backup/actions, fix [772](https://github.com/Altinity/clickhouse-backup/issues/772) + +BUG FIXES +- fixed behavior for `restore_as_attach: true` for non-replicated MergeTree, fix [773](https://github.com/Altinity/clickhouse-backup/issues/773) +- tables with `ENGINE=Dictionary` shall create after all `dictionaries` to avoid retry, fix [771](https://github.com/Altinity/clickhouse-backup/issues/771) + +# v2.4.2 +IMPROVEMENTS +- added `cpu_nice_priority` and `io_nice_priority` to config, which allow us to throttle CPU and IO usage for the whole `clickhouse-backup` process, fix [757](https://github.com/Altinity/clickhouse-backup/issues/757) + +BUG FIXES +- fixed restore for object disk frozen_metadata.txt, fix [752](https://github.com/Altinity/clickhouse-backup/issues/752) +- fixed more corner cases for `check_parts_columns: true`, fix [747](https://github.com/Altinity/clickhouse-backup/issues/747) +- fixed applying macros to s3 endpoint in object disk during restoring embedded backups, fix [750](https://github.com/Altinity/clickhouse-backup/issues/750) +- rewrote `GCS` clients pool, set default `GCS_CLIENT_POOL_SIZE` as `max(upload_concurrency, download_concurrency) * 3` to avoid stuck, fix [753](https://github.com/Altinity/clickhouse-backup/pull/753), thanks @minguyen-jumptrading + +# v2.4.1 +IMPROVEMENTS +- switched to go-1.21 +- added clickhouse-server:23.8 for integration and testflows tests +- added `FTP_SKIP_TLS_VERIFY` config option fix [742](https://github.com/Altinity/clickhouse-backup/issues/742) + +BUG FIXES +- fixed calculation part size for `S3` and buffer size for `Azure` to avoid errors for upload big files, fix [739](https://github.com/Altinity/clickhouse-backup/issues/739) thanks @rodrigargar +- fixed GetFileReader for SSE encryption in `S3`, again fix [709](https://github.com/Altinity/clickhouse-backup/issues/709) + +# v2.4.0 IMPROVEMENTS - first implementation for properly backup S3/GCS/Azure disks, support server-side copy to back up bucket during `clickhouse-backup` create and during `clickhouse-backup restore`, requires add `object_disk_path` to `s3`,`gcs`,`azblob` section, fix [447](https://github.com/Altinity/clickhouse-backup/issues/447) -- Implementation blacklist for table engines during backup / download / upload / restore [537](https://github.com/Altinity/clickhouse-backup/issues/537) +- implementation blocklist for table engines during backup / download / upload / restore [537](https://github.com/Altinity/clickhouse-backup/issues/537) - restore RBAC / configs, refactoring restart clickhouse-server via `sql:SYSTEM SHUTDOWN` or `exec:systemctl restart clickhouse-server`, add `--rbac-only` and `--configs-only` options to `create`, `upload`, `download`, `restore` command. fix [706]https://github.com/Altinity/clickhouse-backup/issues/706 - Backup/Restore RBAC related objects from Zookeeper via direct connection to zookeeper/keeper, fix [604](https://github.com/Altinity/clickhouse-backup/issues/604) -- Add `SHARDED_OPERATION_MODE` option, to easy create backup for sharded cluster, available values `none` (no sharding), `table` (table granularity), `database` (database granularity), `first-replica` (on the lexicographically sorted first active replica), thanks @mskwon, fix [639](https://github.com/Altinity/clickhouse-backup/issues/639), fix [648](https://github.com/Altinity/clickhouse-backup/pull/648) -- Add support for `compression_format: none` for upload and download backups created with `--rbac` / `--rbac-only` or `--configs` / `--configs-only` options, fix [713](https://github.com/Altinity/clickhouse-backup/issues/713) -- Add support for s3 `GLACIER` storage class, when GET return error, then, it requires 5 minutes per key and restore could be slow. Use `GLACIER_IR`, it looks more robust, fix [614](https://github.com/Altinity/clickhouse-backup/issues/614) -- Try Make ./tests/integration/ test parallel fix [721](https://github.com/Altinity/clickhouse-backup/issues/721) - -BUG FIXES -- fix possible create backup failures during UNFREEZE not exists tables, affected 2.2.7+ version, fix [704](https://github.com/Altinity/clickhouse-backup/issues/704) -- fix too strict `system.parts_columns` check when backup create, exclude Enum and Tuple (JSON) and Nullable(Type) vs Type corner cases, fix [685](https://github.com/Altinity/clickhouse-backup/issues/685), fix [699](https://github.com/Altinity/clickhouse-backup/issues/699) -- fix `--rbac` behavior when /var/lib/clickhouse/access not exists -- restore functions via `CREATE OR REPLACE` -- fix `skip_databases` behavior for corner case `--tables="*pattern.*"` -- fix `skip_database_engines` behavior +- added `SHARDED_OPERATION_MODE` option, to easily create backup for sharded cluster, available values `none` (no sharding), `table` (table granularity), `database` (database granularity), `first-replica` (on the lexicographically sorted first active replica), thanks @mskwon, fix [639](https://github.com/Altinity/clickhouse-backup/issues/639), fix [648](https://github.com/Altinity/clickhouse-backup/pull/648) +- added support for `compression_format: none` for upload and download backups created with `--rbac` / `--rbac-only` or `--configs` / `--configs-only` options, fix [713](https://github.com/Altinity/clickhouse-backup/issues/713) +- added support for s3 `GLACIER` storage class, when GET return error, then, it requires 5 minutes per key and restore could be slow. Use `GLACIER_IR`, it looks more robust, fix [614](https://github.com/Altinity/clickhouse-backup/issues/614) +- restore functions via `CREATE OR REPLACE` for more atomic behavior +- prepared to make `./tests/integration/` test parallel execution fix [721](https://github.com/Altinity/clickhouse-backup/issues/721) +- touch `/var/lib/clickhouse/flags/force_drop_table` before every DROP TABLE execution, fix [683](https://github.com/Altinity/clickhouse-backup/issues/683) +- added support connection pool for Google Cloud Storage, `GCS_CLIENT_POOL_SIZE`, fix [724](https://github.com/Altinity/clickhouse-backup/issues/724) + +BUG FIXES +- fixed possible create backup failures during UNFREEZE not exists tables, affected 2.2.7+ version, fix [704](https://github.com/Altinity/clickhouse-backup/issues/704) +- fixed too strict `system.parts_columns` check when backup creates, exclude `Enum` and `Tuple (JSON)` and `Nullable(Type)` vs `Type` corner cases, fix [685](https://github.com/Altinity/clickhouse-backup/issues/685), fix [699](https://github.com/Altinity/clickhouse-backup/issues/699) +- fixed `--rbac` behavior when /var/lib/clickhouse/access not exists +- fixed `skip_database_engines` behavior +- fixed `skip_databases` behavior during restore for corner case `db.prefix*` and corner cases when conflict with `--tables="*pattern.*"`, fix [663](https://github.com/Altinity/clickhouse-backup/issues/663) +- fixed S3 head object Server Side Encryption parameters, fix [709](https://github.com/Altinity/clickhouse-backup/issues/709) # v2.3.2 BUG FIXES @@ -30,7 +395,7 @@ IMPROVEMENTS BUG FIXES - fix static build for FIPS compatible mode fix [693](https://github.com/Altinity/clickhouse-backup/issues/693) -- complete success/failure server callback notification even when main context canceled, fix [680](https://github.com/Altinity/clickhouse-backup/pull/680) +- complete success/failure server callback notification even when the main context canceled, fix [680](https://github.com/Altinity/clickhouse-backup/pull/680) - `clean` command will not return error when shadow directory not exists, fix [686](https://github.com/Altinity/clickhouse-backup/issues/686) # v2.3.0 @@ -42,21 +407,21 @@ IMPROVEMENTS - migrate from `apex/log` to `rs/zerolog`, fix RaceConditions, fix [624](https://github.com/Altinity/clickhouse-backup/issues/624),see details https://github.com/apex/log/issues/103 BUG FIXES -- hotfix wrong empty files when disk_mapping contains not exists during create, affected 2.2.7 version, look details [676](https://github.com/Altinity/clickhouse-backup/issues/676#issue-1771732489) +- hotfix wrong empty files when disk_mapping contains don't exist during creation, affected 2.2.7 version, look details [676](https://github.com/Altinity/clickhouse-backup/issues/676#issue-1771732489) - add `FTP_ADDRESS` and `SFTP_PORT` in Default config Readme.md section fix [668](https://github.com/Altinity/clickhouse-backup/issues/668) - when use `--tables=db.materialized_view` pattern, then create/restore backup also for `.inner.materialized_view` or `.inner_id.uuid`, fix [613](https://github.com/Altinity/clickhouse-backup/issues/613) # v2.2.8 BUG FIXES -- hotfix wrong empty files when disk_mapping contains not exists during create, affected 2.2.7 version, look details [676](https://github.com/Altinity/clickhouse-backup/issues/676#issue-1771732489) +- hotfix wrong empty files when disk_mapping contains don't exist during creation, affected 2.2.7 version, look details [676](https://github.com/Altinity/clickhouse-backup/issues/676#issue-1771732489) # v2.2.7 IMPROVEMENTS -- Auto-tuning concurrency and buffer size related parameters depending on remote storage type, fix [658](https://github.com/Altinity/clickhouse-backup/issues/658) -- add `CLICKHOUSE_BACKUP_MUTATIONS` and `CLICKHOUSE_RESTORE_AS_ATTACH` config options to allow backup and properly restore table with system.mutations is_done=0 status. fix [529](https://github.com/Altinity/clickhouse-backup/issues/529) +- Auto-tuning concurrency and buffer size related parameters depending on the remote storage type, fix [658](https://github.com/Altinity/clickhouse-backup/issues/658) +- add `CLICKHOUSE_BACKUP_MUTATIONS` and `CLICKHOUSE_RESTORE_AS_ATTACH` config options to allow backup and properly restore table with `system.mutations` where is_done=0 status. fix [529](https://github.com/Altinity/clickhouse-backup/issues/529) - add `CLICKHOUSE_CHECK_PARTS_COLUMNS` config option and `--skip-check-parts-column` CLI parameter to `watch`, `create` and `create_remote` commands to disallow backup with inconsistent column data types fix [529](https://github.com/Altinity/clickhouse-backup/issues/529) - add test coverage reports for unit, testflows and integration tests, fix [644](https://github.com/Altinity/clickhouse-backup/issues/644) -- use UNFREEZE TABLE in ClickHouse after backup finished to allow s3 and other object storage disks unlock and delete remote keys during merge, fix [423](https://github.com/Altinity/clickhouse-backup/issues/423) +- use UNFREEZE TABLE in ClickHouse after backup finished to allow s3 and other object storage disks to unlock and delete remote keys during merge, fix [423](https://github.com/Altinity/clickhouse-backup/issues/423) BUG FIXES - apply `SETTINGS check_table_dependencies=0` to `DROP DATABASE` statement, when pass `--ignore-dependencies` together with `--rm` in `restore` command, fix [651](https://github.com/Altinity/clickhouse-backup/issues/651) @@ -65,14 +430,14 @@ BUG FIXES # v2.2.6 BUG FIXES - fix panic for resume upload after restart API server for boolean parameters, fix [653](https://github.com/Altinity/clickhouse-backup/issues/653) -- apply SETTINGS check_table_dependencies=0 to DROP DATABASE statement, when pass `--ignore-dependencies` together with `--rm` in `restore` command, fix [651](https://github.com/Altinity/clickhouse-backup/issues/651) +- apply `SETTINGS check_table_dependencies=0` to `DROP DATABASE` statement, when pass `--ignore-dependencies` together with `--rm` in `restore` command, fix [651](https://github.com/Altinity/clickhouse-backup/issues/651) # v2.2.5 BUG FIXES - fix error after restart API server for boolean parameters, fix [646](https://github.com/Altinity/clickhouse-backup/issues/646) -- fix corner cases when `restore_schema_on_cluster: cluster`, fix [642](https://github.com/Altinity/clickhouse-backup/issues/642), error happens on 2.2.0-2.2.4 +- fix corner cases when `restore_schema_on_cluster: cluster`, fix [642](https://github.com/Altinity/clickhouse-backup/issues/642), error happens from 2.2.0 to 2.2.4 - fix `Makefile` targets `build-docker` and `build-race-docker` for old clickhouse-server version -- fix typo `retries_pause` config definition in general section +- fix typo `retries_pause` config definition in a `general` config section # v2.2.4 BUG FIXES @@ -87,7 +452,7 @@ IMPROVEMENTS - add `callback` parameter to upload, download, create, restore API endpoints, fix [636](https://github.com/Altinity/clickhouse-backup/pull/636) BUG FIXES -- add system.macros could be applied to `path` config section to ReadMe.md, fix [638](https://github.com/Altinity/clickhouse-backup/issues/638) +- add `system.macros` could be applied to `path` config section to ReadMe.md, fix [638](https://github.com/Altinity/clickhouse-backup/issues/638) - fix connection leaks for S3 versioned buckets during execution upload and delete command, fix [637](https://github.com/Altinity/clickhouse-backup/pull/637) # v2.2.1 @@ -107,8 +472,8 @@ IMPROVEMENTS - improved behavior `--partitions` parameter, for cases when PARTITION BY clause return hashed value instead of numeric prefix for `partition_id` in `system.parts`, fix [602](https://github.com/Altinity/clickhouse-backup/issues/602) - apply `system.macros` values when use `restore_schema_on_cluster` and replace cluster name in engine=Distributed tables, fix [574](https://github.com/Altinity/clickhouse-backup/issues/574) - switch S3 storage backend to https://github.com/aws/aws-sdk-go-v2/, fix [534](https://github.com/Altinity/clickhouse-backup/issues/534) -- added `S3_OBJECT_LABLES` and `GCS_OBJECT_LABELS` to allow setup each backup object metadata during upload fix [588](https://github.com/Altinity/clickhouse-backup/issues/588) -- added `clickhouse-keeper` as zookeeper replacement for integration test during reproduce [416](https://github.com/Altinity/clickhouse-backup/issues/416) +- added `S3_OBJECT_LABLES` and `GCS_OBJECT_LABELS` to allow setting each backup object metadata during upload fix [588](https://github.com/Altinity/clickhouse-backup/issues/588) +- added `clickhouse-keeper` as zookeeper replacement for integration test during reproducing [416](https://github.com/Altinity/clickhouse-backup/issues/416) - decrease memory buffers for S3 and GCS, change default value for `upload_concurrency` and `download_concurrency` to `round(sqrt(MAX_CPU / 2))`, fix [539](https://github.com/Altinity/clickhouse-backup/issues/539) - added ability to set up custom storage class for GCS and S3 depends on backupName pattern, fix [584](https://github.com/Altinity/clickhouse-backup/issues/584) @@ -117,10 +482,10 @@ BUG FIXES - fix wrong Content-Type header, fix [605](https://github.com/Altinity/clickhouse-backup/issues/605) - fix wrong behavior for `download` with `--partitions` fix [606](https://github.com/Altinity/clickhouse-backup/issues/606) - fix wrong size of backup in list command if upload or download was break and resume, fix [526](https://github.com/Altinity/clickhouse-backup/issues/526) -- fix `_successful_` and `_failed_` metrics counter issue, happens after 2.1.0, fix [589](https://github.com/Altinity/clickhouse-backup/issues/589) +- fix `_successful_` and `_failed_` issue related to metrics counter, happens after 2.1.0, fix [589](https://github.com/Altinity/clickhouse-backup/issues/589) - fix wrong calculation date of last remote backup during startup - fix wrong duration, status for metrics after 2.1.0 refactoring, fix [599](https://github.com/Altinity/clickhouse-backup/issues/599) -- fix panic on LIVE VIEW tables with option --restore-database-mapping db:db_new enabled), thanks @php53unit +- fix panic on `LIVE VIEW` tables with option `--restore-database-mapping db:db_new` enabled, thanks @php53unit # v2.1.3 IMPROVEMENTS @@ -134,9 +499,9 @@ BUG FIXES - fix wrong `clean` command and API behavior, fix [533](https://github.com/Altinity/clickhouse-backup/issues/533) - fix getMacro usage in Examples for backup / restore sharded cluster. - fix deletion files from S3 versioned bucket, fix [555](https://github.com/Altinity/clickhouse-backup/issues/555) -- fix `--restore-database-mapping` behavior for `ReplicatedMergeTree` (replace database name in replication path) and `Distributed` (replace database name in underlying table) tables, fix [547](https://github.com/Altinity/clickhouse-backup/issues/547) +- fix `--restore-database-mapping` behavior for `ReplicatedMergeTree` (replace database name in a replication path) and `Distributed` (replace database name in underlying table) tables, fix [547](https://github.com/Altinity/clickhouse-backup/issues/547) - `MaterializedPostgreSQL` doesn't support FREEZE, fix [550](https://github.com/Altinity/clickhouse-backup/issues/550), see also https://github.com/ClickHouse/ClickHouse/issues/32902, https://github.com/ClickHouse/ClickHouse/issues/44252 -- `create` and `restore` commands will respect `skip_tables` config options and `--table` cli parameter, to avoid create unnecessary empty databases, fix [583](https://github.com/Altinity/clickhouse-backup/issues/583) +- `create` and `restore` commands will respect `skip_tables` config options and `--table` cli parameter, to avoid creating unnecessary empty databases, fix [583](https://github.com/Altinity/clickhouse-backup/issues/583) - fix `watch` unexpected connection closed behavior, fix [568](https://github.com/Altinity/clickhouse-backup/issues/568) - fix `watch` validation parameters corner cases, close [569](https://github.com/Altinity/clickhouse-backup/pull/569) - fix `--restore-database-mapping` behavior for `ATTACH MATERIALIZED VIEW`, `CREATE VIEW` and `restore --data` corner cases, fix [559](https://github.com/Altinity/clickhouse-backup/issues/559) @@ -151,11 +516,11 @@ BUG FIXES # v2.1.1 BUG FIXES -- return bash and clickhouse usergroup to Dockerfile image short, fix [542](https://github.com/Altinity/clickhouse-backup/issues/542) +- return `bash` and `clickhouse` user group to `Dockerfile` image short, fix [542](https://github.com/Altinity/clickhouse-backup/issues/542) # v2.1.0 IMPROVEMENTS -- complex refactoring to use contexts, AWS and SFTP storage not full supported +- complex refactoring to use contexts, `AWS` and `SFTP` storage are not fully supported - complex refactoring for logging to avoid race condition when change log level during config reload - improve kubernetes example for adjust incremental backup, fix [523](https://github.com/Altinity/clickhouse-backup/issues/523) - add storage independent retries policy, fix [397](https://github.com/Altinity/clickhouse-backup/issues/397) @@ -173,7 +538,7 @@ BUG FIXES - for `restore` command call `DROP DATABASE IF EXISTS db SYNC` when pass `--schema` and `--drop` together, fix [514](https://github.com/Altinity/clickhouse-backup/issues/514) - close persistent connections for remote backup storage after command execution, fix [535](https://github.com/Altinity/clickhouse-backup/issues/535) - lot of typos fixes -- fix all commands was always return 200 status (expect errors) and ignore status which passed from application code in API server +- fix all commands was always return `200` status (expect errors) and ignore status which passed from application code in API server # v2.0.0 IMPROVEMENTS @@ -184,8 +549,8 @@ IMPROVEMENTS - switch to go 1.19 - refactoring to remove legacy `storage` package - add `table` parameter to `tables` cli command and `/backup/tables` API handler, fix [367](https://github.com/Altinity/clickhouse-backup/issues/367) -- add `--resumable` parameter to `create_remote`, `upload`, `restore_remote`, `donwload` commands to allow resume upload or download after break. Ignored for `remote_storage: custom`, fix [207](https://github.com/Altinity/clickhouse-backup/issues/207) -- add `--ignore-dependencies` parameter to `restore` and `restore_remote`, to allow drop object during restore schema on server where schema objects already exists and contains dependencies which not present in backup, fix [455](https://github.com/Altinity/clickhouse-backup/issues/455) +- add `--resumable` parameter to `create_remote`, `upload`, `restore_remote`, `donwload` commands to allow resuming upload or download after break. Ignored for `remote_storage: custom`, fix [207](https://github.com/Altinity/clickhouse-backup/issues/207) +- add `--ignore-dependencies` parameter to `restore` and `restore_remote`, to allow drop object during restore schema on server where schema objects already exist and contain dependencies which not present in backup, fix [455](https://github.com/Altinity/clickhouse-backup/issues/455) - add `restore --restore-database-mapping=:[,<...>]`, fix [269](https://github.com/Altinity/clickhouse-backup/issues/269), thanks @mojerro BUG FIXES @@ -218,7 +583,7 @@ BUG FIXES # v1.5.0 IMPROVEMENTS -- try to improve implementation `check_replicas_before_attach` configuration to avoid concurrent ATTACH PART execution during `restore` command on multi-shard cluster, fix [474](https://github.com/Altinity/clickhouse-backup/issues/474) +- try to improve implementation `check_replicas_before_attach` configuration to avoid concurrently ATTACH PART execution during `restore` command on multi-shard cluster, fix [474](https://github.com/Altinity/clickhouse-backup/issues/474) - add `timeout` to `azure` config `AZBLOB_TIMEOUT` to allow download with bad network quality, fix [467](https://github.com/Altinity/clickhouse-backup/issues/467) # v1.4.9 @@ -228,7 +593,7 @@ BUG FIXES # v1.4.8 IMPROVEMENTS - add `check_replicas_before_attach` configuration to avoid concurrent ATTACH PART execution during `restore` command on multi-shard cluster, fix [474](https://github.com/Altinity/clickhouse-backup/issues/474) -- allow backup list when clickhouse server offline, fix [476](https://github.com/Altinity/clickhouse-backup/issues/476) +- allow a backup list when clickhouse server offline, fix [476](https://github.com/Altinity/clickhouse-backup/issues/476) - add `use_custom_storage_class` (`S3_USE_CUSTOM_STORAGE_CLASS`) option to `s3` section, thanks @realwhite BUG FIXES @@ -236,23 +601,23 @@ BUG FIXES # v1.4.7 IMPROVEMENTS -- PROPERLY restore to default disk if disks not found on destination clickhouse server, fix [457](https://github.com/Altinity/clickhouse-backup/issues/457) +- PROPERLY restore to default disk if disks are not found on destination clickhouse server, fix [457](https://github.com/Altinity/clickhouse-backup/issues/457) # v1.4.6 BUG FIXES -- fix infinite loop `error can't acquire semaphore during Download: context canceled`, and `error can't acquire semaphore during Upload: context canceled` all 1.4.x users recommends upgrade to 1.4.6 +- fix infinite loop `error can't acquire semaphore during Download: context canceled`, and `error can't acquire semaphore during Upload: context canceled` all 1.4.x users recommend upgrade to 1.4.6 # v1.4.5 IMPROVEMENTS -- add `CLICKHOUSE_FREEZE_BY_PART_WHERE` option which allow freeze by part with WHERE condition, thanks @vahid-sohrabloo +- add `CLICKHOUSE_FREEZE_BY_PART_WHERE` option which allow freezes by part with WHERE condition, thanks @vahid-sohrabloo # v1.4.4 IMPROVEMENTS -- download and restore to default disk if disks not found on destination clickhouse server, fix [457](https://github.com/Altinity/clickhouse-backup/issues/457) +- download and restore to default disk if disks are not found on destination clickhouse server, fix [457](https://github.com/Altinity/clickhouse-backup/issues/457) # v1.4.3 IMPROVEMENTS -- add `API_INTEGRATION_TABLES_HOST` option to allow use DNS name in integration tables system.backup_list, system.backup_actions +- add `API_INTEGRATION_TABLES_HOST` option to allow using DNS name in integration tables system.backup_list, system.backup_actions BUG FIXES - fix `upload_by_part: false` max file size calculation, fix [454](https://github.com/Altinity/clickhouse-backup/issues/454) @@ -285,7 +650,7 @@ IMPROVEMENTS - add multithreading GZIP implementation BUG FIXES -- fix [406](https://github.com/Altinity/clickhouse-backup/issues/406), properly handle `path` for S3, GCS for case when it begins from "/" +- fix [406](https://github.com/Altinity/clickhouse-backup/issues/406), properly handle `path` for S3, GCS for cases when it begins from "/" - fix [409](https://github.com/Altinity/clickhouse-backup/issues/409), avoid delete partially uploaded backups via `backups_keep_remote` option - fix [422](https://github.com/Altinity/clickhouse-backup/issues/422), avoid cache broken (partially uploaded) remote backup metadata. - fix [404](https://github.com/Altinity/clickhouse-backup/issues/404), properly calculate S3_PART_SIZE to avoid freeze after 10000 multi parts uploading, properly handle error when upload and download go-routine failed to avoid pipe stuck @@ -296,7 +661,7 @@ IMPROVEMENTS - fix [387](https://github.com/Altinity/clickhouse-backup/issues/387#issuecomment-1034648447), improve documentation related to memory and CPU usage BUG FIXES -- fix [392](https://github.com/Altinity/clickhouse-backup/issues/392), correct download for recursive sequence of diff backups when `DOWNLOAD_BY_PART` true +- fix [392](https://github.com/Altinity/clickhouse-backup/issues/392), correct download for a recursive sequence of diff backups when `DOWNLOAD_BY_PART` true - fix [390](https://github.com/Altinity/clickhouse-backup/issues/390), respect skip_tables patterns during restore and skip all INFORMATION_SCHEMA related tables even skip_tables don't contain INFORMATION_SCHEMA pattern - fix [388](https://github.com/Altinity/clickhouse-backup/issues/388), improve restore ON CLUSTER for VIEW with TO clause - fix [385](https://github.com/Altinity/clickhouse-backup/issues/385), properly handle multiple incremental backup sequences + `BACKUPS_TO_KEEP_REMOTE` @@ -311,11 +676,11 @@ IMPROVEMENTS - Add implementation `--diff-from-remote` for `upload` command and properly handle `required` on download command, fix [#289](https://github.com/Altinity/clickhouse-backup/issues/289) - Add `print-config` cli command fix [#366](https://github.com/Altinity/clickhouse-backup/issues/366) - Add `UPLOAD_BY_PART` (default: true) option for improve upload/download concurrency fix [#324](https://github.com/Altinity/clickhouse-backup/issues/324) -- Add support ARM platform for Docker images and pre-compiled binary files, fix [#312](https://github.com/Altinity/clickhouse-backup/issues/312) +- Add a support ARM platform for Docker images and pre-compiled binary files, fix [#312](https://github.com/Altinity/clickhouse-backup/issues/312) - KeepRemoteBackups should respect differential backups, fix [#111](https://github.com/Altinity/clickhouse-backup/issues/111) - Add `SFTP_DEBUG` option, fix [#335](https://github.com/Altinity/clickhouse-backup/issues/335) -- Add ability to restore schema `ON CLUSTER`, fix [#145](https://github.com/Altinity/clickhouse-backup/issues/145) -- Add support encrypted disk (include s3 encrypted disks), fix [#260](https://github.com/Altinity/clickhouse-backup/issues/260) +- Add the ability to restore schema `ON CLUSTER`, fix [#145](https://github.com/Altinity/clickhouse-backup/issues/145) +- Add support for encrypted disk (include s3 encrypted disks), fix [#260](https://github.com/Altinity/clickhouse-backup/issues/260) - API Server optimization for speed of `last_backup_size_remote` metric calculation to make it async during REST API startup and after download/upload, fix [#309](https://github.com/Altinity/clickhouse-backup/issues/309) - Improve `list remote` speed via local metadata cache in `$TEMP/.clickhouse-backup.$REMOTE_STORAGE`, fix [#318](https://github.com/Altinity/clickhouse-backup/issues/318) @@ -334,8 +699,8 @@ BUG FIXES container via entrypoint - fix [#317](https://github.com/Altinity/clickhouse-backup/issues/317), properly upload large files to Azure Blob Storage -- fix [#220](https://github.com/Altinity/clickhouse-backup/issues/220), properly handle total_bytes for uint64 type -- fix [#304](https://github.com/Altinity/clickhouse-backup/issues/304), properly handle archive extension during download instead of use config settings +- fix [#220](https://github.com/Altinity/clickhouse-backup/issues/220), properly handle total_bytes for `uint64` type +- fix [#304](https://github.com/Altinity/clickhouse-backup/issues/304), properly handle an archive extension during download instead of use config settings - fix [#375](https://github.com/Altinity/clickhouse-backup/issues/375), properly `REMOTE_STORAGE=none` error handle - fix [#379](https://github.com/Altinity/clickhouse-backup/issues/379), will try to clean `shadow` if `create` fail during `moveShadow` - more precise calculation backup size during `upload`, for backups created with `--partitions`, fix bug after [#356](https://github.com/Altinity/clickhouse-backup/pull/356) @@ -382,11 +747,11 @@ BUG FIXES # v1.2.0 INCOMPATIBLE CHANGES -- REST API `/backup/status` now return only latest executed command with status and error message +- REST API `/backup/status` now return only the latest executed command with a status and error message IMPROVEMENTS -- Added REST API `/backup/list/local` and `/backup/list/remote` to allow list backup types separately -- Decreased background backup creation time via REST API `/backup/create`, during avoid list remote backups for update metrics value +- Added REST API `/backup/list/local` and `/backup/list/remote` to allow listing backup types separately +- Decreased background backup creation time via REST API `/backup/create`, during avoiding listing remote backups for update metrics value - Decreased backup creation time, during avoid scan whole `system.tables` when set `table` query string parameter or `--tables` cli parameter - Added `last` and `filter` query string parameters to REST API `/backup/actions`, to avoid pass to client long JSON documents - Improved `FTP` remote storage parallel upload / download @@ -402,7 +767,7 @@ BUG FIXES - fix [#252](https://github.com/Altinity/clickhouse-backup/issues/252), now you can pass `last` and `filter` query string parameters - fix [#246](https://github.com/Altinity/clickhouse-backup/issues/246), incorrect error messages when use `REMOTE_STORAGE=none` - fix [#283](https://github.com/Altinity/clickhouse-backup/issues/283), properly handle error message from `FTP` server -- fix [#268](https://github.com/Altinity/clickhouse-backup/issues/268), properly restore legacy backup for schema without database name +- fix [#268](https://github.com/Altinity/clickhouse-backup/issues/268), properly restore legacy backup for schema without a database name # v1.1.1 @@ -420,7 +785,7 @@ IMPROVEMENTS - Now `S3_PART_SIZE` not restrict upload size, partSize calculate depends on `MAX_FILE_SIZE` - improve logging for delete operation - Added `S3_DEBUG` option to allow debug S3 connection -- Decrease number of SQL queries to system.* during backup commands +- Decrease the number of SQL queries to `system.*` during backup commands - Added options for RBAC and CONFIG backup, look to `clickhouse-backup help create` and `clickhouse-backup help restore` for details - Add `S3_CONCURRENCY` option to speedup backup upload to `S3` - Add `SFTP_CONCURRENCY` option to speedup backup upload to `SFTP` @@ -431,10 +796,10 @@ IMPROVEMENTS - Add `CLICKHOUSE_DEBUG` option to allow low-level debug for `clickhouse-go` BUG FIXES -- fix [#266](https://github.com/Altinity/clickhouse-backup/discussions/266) properly restore legacy backup format +- fix [#266](https://github.com/Altinity/clickhouse-backup/discussions/266) properly restore a legacy backup format - fix [#244](https://github.com/Altinity/clickhouse-backup/issues/244) add `read_timeout`, `write_timeout` to client-side timeout for `clickhouse-go` - fix [#255](https://github.com/Altinity/clickhouse-backup/issues/255) restrict connection pooling to 1 in `clickhouse-go` -- fix [#256](https://github.com/Altinity/clickhouse-backup/issues/256) remote_storage: none, was broke compression +- fix [#256](https://github.com/Altinity/clickhouse-backup/issues/256) `remote_storage: none`, was broke compression - fix [#266](https://github.com/Altinity/clickhouse-backup/discussions/266) legacy backups from version prior 1.0 can't restore without `allow_empty_backup: true` - fix [#223](https://github.com/Altinity/clickhouse-backup/issues/223) backup only database metadata for proxy integrated database engines like MySQL, PostgresSQL - fix `GCS` global buffer wrong usage during UPLOAD_CONCURRENCY > 1 @@ -443,7 +808,7 @@ BUG FIXES # v1.0.0 BUG FIXES -- Fixed silent cancel uploading when table has more than 4k files (fix [#203](https://github.com/Altinity/clickhouse-backup/issues/203), [#163](https://github.com/Altinity/clickhouse-backup/issues/163). Thanks [mastertheknife](https://github.com/mastertheknife)) +- Fixed silent cancel uploading when table has more than 4k files, fix [#203](https://github.com/Altinity/clickhouse-backup/issues/203), [#163](https://github.com/Altinity/clickhouse-backup/issues/163). Thanks [mastertheknife](https://github.com/mastertheknife) - Fixed download error for `zstd` and `brotli` compression formats - Fixed bug when old-format backups hadn't cleared @@ -456,7 +821,7 @@ IMPROVEMENTS - Added SFTP remote storage (Thanks [@combin](https://github.com/combin)) - Now databases will be restored with the same engines (Thanks [@Slach](https://github.com/Slach)) - Added `create_remote` and `restore_remote` commands -- Changed of compression format list. Added `zstd`, `brotli` and disabled `bzip2`, `sz`, `xz` +- Changed of a compression format list. Added `zstd`, `brotli` and disabled `bzip2`, `sz`, `xz` BUG FIXES - Fixed empty backup list when S3_PATH and AZBLOB_PATH is root @@ -496,11 +861,11 @@ IMPROVEMENTS - Ability to upload and download specific tables from backup - Added partitions backup on remote storage ([#83](https://github.com/Altinity/clickhouse-backup/issues/83)) - Added support for backup/upload/download schema only ([#138](https://github.com/Altinity/clickhouse-backup/issues/138)) -- Added new backup format select it by `compression_format: none` option +- Added a new backup format select it by `compression_format: none` option BROKEN CHANGES -- Changed backup format -- Incremental backup on remote storage feature is not supported now, but will support in future versions +- Changed a backup format +- Incremental backup on remote storage feature is not supported now, but will be supported in future versions # v0.6.4 diff --git a/Dockerfile b/Dockerfile index 3accc2bb..39a951c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,22 +8,23 @@ USER root RUN rm -fv /etc/apt/sources.list.d/clickhouse.list && \ find /etc/apt/ -type f -name *.list -exec sed -i 's/ru.archive.ubuntu.com/archive.ubuntu.com/g' {} + && \ ( apt-get update || true ) && \ - apt-get install -y --no-install-recommends gnupg ca-certificates wget && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 52B59B1571A79DBC054901C0F6BC817356A3D45E && \ + apt-get install -y --no-install-recommends gnupg ca-certificates wget && update-ca-certificates && \ + for srv in "keyserver.ubuntu.com" "pool.sks-keyservers.net" "keys.gnupg.net"; do host $srv; apt-key adv --keyserver $srv --recv-keys 52B59B1571A79DBC054901C0F6BC817356A3D45E; if [ $? -eq 0 ]; then break; fi; done && \ DISTRIB_CODENAME=$(cat /etc/lsb-release | grep DISTRIB_CODENAME | cut -d "=" -f 2) && \ echo ${DISTRIB_CODENAME} && \ echo "deb https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu ${DISTRIB_CODENAME} main" > /etc/apt/sources.list.d/golang.list && \ echo "deb-src https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu ${DISTRIB_CODENAME} main" >> /etc/apt/sources.list.d/golang.list && \ ( apt-get update || true ) && \ - apt-get install -y --no-install-recommends libc-dev golang-1.21 make git gcc musl-dev musl-tools && \ + apt-get install -y --no-install-recommends libc-dev golang-1.22 make git gcc musl-dev musl-tools && \ wget -q -P /root/ https://musl.cc/aarch64-linux-musl-cross.tgz && \ tar -xvf /root/aarch64-linux-musl-cross.tgz -C /root/ && \ mkdir -p /root/go/ -RUN ln -nsfv /usr/lib/go-1.21/bin/go /usr/bin/go +RUN ln -nsfv /usr/lib/go-1.22/bin/go /usr/bin/go VOLUME /root/.cache/go ENV GOCACHE=/root/.cache/go ENV GOPATH=/root/go/ -ENV GOROOT=/usr/lib/go-1.21/ +ENV GOROOT=/usr/lib/go-1.22/ RUN go env WORKDIR /src/ # cache modules when go.mod go.sum changed @@ -75,9 +76,18 @@ COPY --from=builder-fips /src/build/ /src/build/ CMD /src/build/${TARGETPLATFORM}/clickhouse-backup-fips --help -FROM alpine:3.18 AS image_short +FROM alpine:3.19 AS image_short ARG TARGETPLATFORM +ARG VERSION=unknown MAINTAINER Eugene Klimov +LABEL "org.opencontainers.image.version"=${VERSION} +LABEL "org.opencontainers.image.vendor"="Altinity Inc." +LABEL "org.opencontainers.image.licenses"="MIT" +LABEL "org.opencontainers.image.title"="Altinity Backup for ClickHouse" +LABEL "org.opencontainers.image.description"="A tool for easy ClickHouse backup and restore with support for many cloud and non-cloud storage types." +LABEL "org.opencontainers.image.source"="https://github.com/Altinity/clickhouse-backup" +LABEL "org.opencontainers.image.documentation"="https://github.com/Altinity/clickhouse-backup/blob/master/Manual.md" + RUN addgroup -S -g 101 clickhouse \ && adduser -S -h /var/lib/clickhouse -s /bin/bash -G clickhouse -g "ClickHouse server" -u 101 clickhouse RUN apk update && apk add --no-cache ca-certificates tzdata bash curl && update-ca-certificates @@ -85,6 +95,7 @@ COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh COPY build/${TARGETPLATFORM}/clickhouse-backup /bin/clickhouse-backup RUN chmod +x /bin/clickhouse-backup +# RUN apk add --no-cache libcap-setcap libcap-getcap && setcap cap_sys_nice=+ep /bin/clickhouse-backup # USER clickhouse ENTRYPOINT ["/entrypoint.sh"] CMD [ "/bin/clickhouse-backup", "--help" ] @@ -92,20 +103,38 @@ CMD [ "/bin/clickhouse-backup", "--help" ] FROM image_short AS image_fips ARG TARGETPLATFORM +ARG VERSION=unknown +LABEL "org.opencontainers.image.version"=${VERSION} +LABEL "org.opencontainers.image.vendor"="Altinity Inc." +LABEL "org.opencontainers.image.licenses"="MIT" +LABEL "org.opencontainers.image.title"="Altinity Backup for ClickHouse" +LABEL "org.opencontainers.image.description"="A tool for easy ClickHouse backup and restore with support for many cloud and non-cloud storage types." +LABEL "org.opencontainers.image.source"="https://github.com/Altinity/clickhouse-backup" +LABEL "org.opencontainers.image.documentation"="https://github.com/Altinity/clickhouse-backup/blob/master/Manual.md" + MAINTAINER Eugene Klimov COPY build/${TARGETPLATFORM}/clickhouse-backup-fips /bin/clickhouse-backup RUN chmod +x /bin/clickhouse-backup +# RUN apk add --no-cache libcap-setcap libcap-getcap && setcap cap_sys_nice=+ep /bin/clickhouse-backup FROM ${CLICKHOUSE_IMAGE}:${CLICKHOUSE_VERSION} AS image_full ARG TARGETPLATFORM +ARG VERSION=unknown MAINTAINER Eugene Klimov - -RUN apt-get update && apt-get install -y gpg && wget -qO- https://kopia.io/signing-key | gpg --dearmor -o /usr/share/keyrings/kopia-keyring.gpg && \ - echo "deb [signed-by=/usr/share/keyrings/kopia-keyring.gpg] http://packages.kopia.io/apt/ stable main" > /etc/apt/sources.list.d/kopia.list && \ +LABEL "org.opencontainers.image.version"=${VERSION} +LABEL "org.opencontainers.image.vendor"="Altinity Inc." +LABEL "org.opencontainers.image.licenses"="MIT" +LABEL "org.opencontainers.image.title"="Altinity Backup for ClickHouse" +LABEL "org.opencontainers.image.description"="A tool for easy ClickHouse backup and restore with support for many cloud and non-cloud storage types." +LABEL "org.opencontainers.image.source"="https://github.com/Altinity/clickhouse-backup" +LABEL "org.opencontainers.image.documentation"="https://github.com/Altinity/clickhouse-backup/blob/master/Manual.md" + +RUN apt-get update && apt-get install -y gpg xxd bsdmainutils parallel && wget -qO- https://kopia.io/signing-key | gpg --dearmor -o /usr/share/keyrings/kopia-keyring.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/kopia-keyring.gpg] https://packages.kopia.io/apt/ stable main" > /etc/apt/sources.list.d/kopia.list && \ wget -c "https://github.com/mikefarah/yq/releases/latest/download/yq_linux_$(dpkg --print-architecture)" -O /usr/bin/yq && chmod +x /usr/bin/yq && \ apt-get update -y && \ - apt-get install -y ca-certificates tzdata bash curl restic rsync rclone jq gpg kopia && \ + apt-get install -y ca-certificates tzdata bash curl restic rsync rclone jq gpg kopia libcap2-bin && \ update-ca-certificates && \ rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/* @@ -113,8 +142,7 @@ COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh COPY build/${TARGETPLATFORM}/clickhouse-backup /bin/clickhouse-backup RUN chmod +x /bin/clickhouse-backup - +# RUN apk add --no-cache libcap-setcap libcap-getcap && setcap cap_sys_nice=+ep /bin/clickhouse-backup # USER clickhouse - ENTRYPOINT ["/entrypoint.sh"] CMD [ "/bin/clickhouse-backup", "--help" ] diff --git a/Examples.md b/Examples.md index 1002acbc..395c5472 100644 --- a/Examples.md +++ b/Examples.md @@ -1,13 +1,13 @@ # Use cases of clickhouse-backup ## How to convert MergeTree to ReplicatedMergeTree -don't work for tables which created in `MergeTree(date_column, (primary keys columns), 8192)` format +This doesn't work for tables created in `MergeTree(date_column, (primary keys columns), 8192)` format 1. Create backup ``` clickhouse-backup create --table='my_db.my_table' my_backup ``` 2. Edit `/var/lib/clickhouse/backup/my_backup/metadata/my_db/my_table.json`, change `query` field, - replace MergeTree() to ReplicatedMergeTree() with parameters according to https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables + replace MergeTree() with ReplicatedMergeTree() with parameters according to https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables 3. Drop table in Clickhouse ``` clickhouse-client -q "DROP TABLE my_db.my.table NO DELAY" @@ -18,37 +18,34 @@ don't work for tables which created in `MergeTree(date_column, (primary keys col ``` ## How to store backups on NFS, backup drive or another server via SFTP -Use 'rsync' -'rsync' supports hard links with means that backup on remote server or mounted fs will be stored as efficiently as in the '/var/lib/clickhouse/backup'. -You can create daily backup by clickhouse-backup and sync backup folder to mounted fs with this command: -`rsync -a -H --delete --progress --numeric-ids --update /var/lib/clickhouse/backup/ /mnt/data/clickhouse-backup/` or similar for sync over ssh. In this case rsync will copy only difference between backups. +Use `rsync`. +`rsync` supports hard links, which means that a backup on a remote server or mounted fs will be stored as efficiently as in `/var/lib/clickhouse/backup`. +You can create a daily backup by clickhouse-backup and a sync backup folder to mounted fs with this command: +`rsync -a -H --delete --progress --numeric-ids --update /var/lib/clickhouse/backup/ /mnt/data/clickhouse-backup/` or similar for sync over ssh. In this case `rsync` will copy only difference between backups. -## How to move data to another clickhouse server -destination server +## How to move data to another ClickHouse server +On the destination server: ```bash mkdir -p /var/lib/clickhouse/backups/backup_name ``` -source server +On the source server: ```bash clickhouse-backup create backup_name rsync --rsh=ssh /var/lib/clickhouse/backups/backup_name/ user@dst_server:/var/lib/clickhouse/backups/backup_name ``` -destination server +On the destination server: ```bash clickhouse-backup restore --rm backup_name ``` -## How to reduce number of partitions -... - -## How to monitor that backups created and uploaded correctly +## How to monitor that backups were created and uploaded correctly Use services like https://healthchecks.io or https://deadmanssnitch.com. -Or use `clickhouse-backup server` and prometheus endpoint :7171/metrics, look alerts examples on https://github.com/Altinity/clickhouse-operator/blob/master/deploy/prometheus/prometheus-alert-rules-backup.yaml +Or use `clickhouse-backup server` and prometheus endpoint `:7171/metrics`. For an example of setting up Prometheus alerts, see https://github.com/Altinity/clickhouse-operator/blob/master/deploy/prometheus/prometheus-alert-rules-backup.yaml. -## How to make backup / restore sharded cluster +## How to back up / restore a sharded cluster ### BACKUP -run only on the first replica for each shard +Run only on the first replica for each shard: ```bash shard_number=$(clickhouse-client -q "SELECT getMacro('shard')") clickhouse-backup create_remote shard${shard_number}-backup @@ -56,22 +53,22 @@ clickhouse-backup delete local shard${shard_number}-backup ``` ### RESTORE -run on all replicas +Run on all replicas: ```bash shard_number=$(clickhouse-client -q "SELECT getMacro('shard')") clickhouse-backup restore_remote --rm --schema shard${shard_number}-backup clickhouse-backup delete local shard${shard_number}-backup ``` -after it, run only on the first replica for each shard +After that, run only on the first replica for each shard: ```bash shard_number=$(clickhouse-client -q "SELECT getMacro('shard')") clickhouse-backup restore_remote --rm shard${shard_number}-backup clickhouse-backup delete local shard${shard_number}-backup ``` -## How to make backup sharded cluster with Ansible -On the first day of month full backup will be uploaded and increments on the others days. -`hosts: clickhouse-cluster` shall be only first replica on each shard +## How to back up a sharded cluster with Ansible +On the first day of month a full backup will be uploaded and increments on the other days. +`hosts: clickhouse-cluster` shall be only the first replica on each shard ```yaml - hosts: clickhouse-cluster @@ -110,12 +107,12 @@ On the first day of month full backup will be uploaded and increments on the oth - uri: url="https://hc-ping.com/{{ healthchecksio_clickhouse_upload_id }}/fail" ``` -## How to make backup database with several terabytes of data +## How to back up a database with several terabytes of data You can use clickhouse-backup for creating periodical backups and keep it local. It protects you from destructive operations. -In addition, you may create instance of ClickHouse on another DC and have it fresh by clickhouse-copier it protects you from hardware or DC failures. +In addition, you may create instance of ClickHouse on another DC and have it fresh by clickhouse-copier to protect you from hardware or DC failures. ## How to use clickhouse-backup in Kubernetes -Install [clickhouse kubernetes operator](https://github.com/Altinity/clickhouse-operator/) and use following manifest +Install the [clickhouse kubernetes operator](https://github.com/Altinity/clickhouse-operator/) and use the following manifest: ```yaml apiVersion: "clickhouse.altinity.com/v1" @@ -232,7 +229,8 @@ spec: containerPort: 7171 ``` -You need to prepare remote storage, for test only +You need to prepare remote storage for test only: + ```yaml --- apiVersion: "apps/v1" @@ -280,7 +278,8 @@ spec: targetPort: minio ``` -Also, you can apply CronJob to run `clickhouse-backup` actions by schedule +You can also use CronJob to run `clickhouse-backup` actions on a schedule: + ```yaml apiVersion: batch/v1 kind: CronJob @@ -315,7 +314,7 @@ spec: value: backup - name: BACKUP_PASSWORD value: "backup_password" - # change to 1, if you want make full backup only in $FULL_BACKUP_WEEKDAY (1 - Mon, 7 - Sun) + # change to 1, if you want to make full backup only in $FULL_BACKUP_WEEKDAY (1 - Mon, 7 - Sun) - name: MAKE_INCREMENT_BACKUP value: "1" - name: FULL_BACKUP_WEEKDAY @@ -332,8 +331,8 @@ spec: fi; for SERVER in $CLICKHOUSE_SERVICES; do if [[ "1" == "$MAKE_INCREMENT_BACKUP" ]]; then - LAST_FULL_BACKUP=$(clickhouse-client -q "SELECT name FROM system.backup_list WHERE location='remote' AND name LIKE '%full%' AND desc NOT LIKE 'broken%' ORDER BY created DESC LIMIT 1 FORMAT TabSeparatedRaw" --host="$SERVER" --port="$CLICKHOUSE_PORT" --user="$BACKUP_USER" $BACKUP_PASSWORD); - TODAY_FULL_BACKUP=$(clickhouse-client -q "SELECT name FROM system.backup_list WHERE location='remote' AND name LIKE '%full%' AND desc NOT LIKE 'broken%' AND toDate(created) = today() ORDER BY created DESC LIMIT 1 FORMAT TabSeparatedRaw" --host="$SERVER" --port="$CLICKHOUSE_PORT" --user="$BACKUP_USER" $BACKUP_PASSWORD) + LAST_FULL_BACKUP=$(clickhouse-client -q "SELECT name FROM system.backup_list WHERE location='remote' AND name LIKE '%${SERVER}%' AND name LIKE '%full%' AND desc NOT LIKE 'broken%' ORDER BY created DESC LIMIT 1 FORMAT TabSeparatedRaw" --host="$SERVER" --port="$CLICKHOUSE_PORT" --user="$BACKUP_USER" $BACKUP_PASSWORD); + TODAY_FULL_BACKUP=$(clickhouse-client -q "SELECT name FROM system.backup_list WHERE location='remote' AND name LIKE '%${SERVER}%' AND name LIKE '%full%' AND desc NOT LIKE 'broken%' AND toDate(created) = today() ORDER BY created DESC LIMIT 1 FORMAT TabSeparatedRaw" --host="$SERVER" --port="$CLICKHOUSE_PORT" --user="$BACKUP_USER" $BACKUP_PASSWORD) PREV_BACKUP_NAME=$(clickhouse-client -q "SELECT name FROM system.backup_list WHERE location='remote' AND desc NOT LIKE 'broken%' ORDER BY created DESC LIMIT 1 FORMAT TabSeparatedRaw" --host="$SERVER" --port="$CLICKHOUSE_PORT" --user="$BACKUP_USER" $BACKUP_PASSWORD); DIFF_FROM[$SERVER]=""; if [[ ("$FULL_BACKUP_WEEKDAY" == "$(date +%u)" && "" == "$TODAY_FULL_BACKUP") || "" == "$PREV_BACKUP_NAME" || "" == "$LAST_FULL_BACKUP" ]]; then @@ -381,7 +380,8 @@ spec: echo "BACKUP CREATED" ``` -For one time restore data you could use `Job` +For one time restore data, you can use `Job`: + ```yaml # example to restore latest backup apiVersion: batch/v1 @@ -425,15 +425,14 @@ spec: CLICKHOUSE_SCHEMA_RESTORE_SERVICES=$(echo $CLICKHOUSE_SCHEMA_RESTORE_SERVICES | tr "," " "); CLICKHOUSE_DATA_RESTORE_SERVICES=$(echo $CLICKHOUSE_DATA_RESTORE_SERVICES | tr "," " "); for SERVER in $CLICKHOUSE_SCHEMA_RESTORE_SERVICES; do - LATEST_BACKUP_NAME=$(clickhouse-client -q "SELECT name FROM system.backup_list WHERE location='remote' AND desc NOT LIKE 'broken%' ORDER BY created DESC LIMIT 1 FORMAT TabSeparatedRaw" --host="$SERVER" --port="$CLICKHOUSE_PORT" --user="$BACKUP_USER" $BACKUP_PASSWORD); + SHARDED_PREFIX=${SERVER%-*} + LATEST_BACKUP_NAME=$(clickhouse-client -q "SELECT name FROM system.backup_list WHERE location='remote' AND desc NOT LIKE 'broken%' AND name LIKE '%${SHARDED_PREFIX}%' ORDER BY created DESC LIMIT 1 FORMAT TabSeparatedRaw" --host="$SERVER" --port="$CLICKHOUSE_PORT" --user="$BACKUP_USER" $BACKUP_PASSWORD); if [[ "" == "$LATEST_BACKUP_NAME" ]]; then echo "Remote backup not found for $SERVER"; exit 1; fi; BACKUP_NAMES[$SERVER]="$LATEST_BACKUP_NAME"; clickhouse-client -mn --echo -q "INSERT INTO system.backup_actions(command) VALUES('restore_remote --schema --rm ${BACKUP_NAMES[$SERVER]}')" --host="$SERVER" --port="$CLICKHOUSE_PORT" --user="$BACKUP_USER" $BACKUP_PASSWORD; - done; - for SERVER in $CLICKHOUSE_SCHEMA_RESTORE_SERVICES; do while [[ "in progress" == $(clickhouse-client -mn -q "SELECT status FROM system.backup_actions WHERE command='restore_remote --schema --rm ${BACKUP_NAMES[$SERVER]}' ORDER BY start DESC LIMIT 1 FORMAT TabSeparatedRaw" --host="$SERVER" --port="$CLICKHOUSE_PORT" --user="$BACKUP_USER" $BACKUP_PASSWORD) ]]; do echo "still in progress ${BACKUP_NAMES[$SERVER]} on $SERVER"; sleep 1; @@ -472,20 +471,25 @@ spec: ## How to use AWS IRSA and IAM to allow S3 backup without Explicit credentials -Create Role and IAM Policy, look details in https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html +Create Role and IAM Policy. This field typically looks like this: +`arn:aws:iam::1393332413596:role/rolename-clickhouse-backup`, +where `1393332413596` is the ID of the role and +`rolename-clickhouse-backup` is the name of the role. +See [the AWS documentation](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) +for all the details. -Create service account with annotations +Create a service account with annotations: ```yaml apiVersion: v1 kind: ServiceAccount metadata: - name: + name: namespace: annotations: eks.amazonaws.com/role-arn: arn:aws:iam:::role/ ``` -Link service account to podTemplate it will create `AWS_ROLE_ARN` and `AWS_WEB_IDENTITY_TOKEN_FILE` environment variables. +Link the service account to a podTemplate to create `AWS_ROLE_ARN` and `AWS_WEB_IDENTITY_TOKEN_FILE` environment variables: ```yaml apiVersion: "clickhouse.altinity.com/v1" kind: "ClickHouseInstallation" @@ -507,9 +511,10 @@ spec: ### How to use clickhouse-backup + clickhouse-operator in FIPS compatible mode in Kubernetes for S3 -use `altinity/clickhouse-backup:X.X.X-fips` as image (where X.X.X version number) -run following commands to generate self-signed TLS keys for secure clickhouse-backup API endpoint, -you need periodically renew this certs, use https://github.com/cert-manager/cert-manager for it in kubernetes +Use the image `altinity/clickhouse-backup:X.X.X-fips` (where X.X.X is the version number). +Run the following commands to generate self-signed TLS keys for secure clickhouse-backup API endpoint: +(You need to renew these certs periodically; use https://github.com/cert-manager/cert-manager for it in kubernetes.) + ```bash openssl genrsa -out ca-key.pem 4096 openssl req -subj "/O=altinity" -x509 -new -nodes -key ca-key.pem -sha256 -days 365000 -out ca-cert.pem @@ -518,7 +523,8 @@ you need periodically renew this certs, use https://github.com/cert-manager/cert openssl x509 -req -days 365 -extensions SAN -extfile <(printf "\n[SAN]\nsubjectAltName=DNS:localhost,DNS:*.cluster.local") -in server-req.csr -out server-cert.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial ``` -create following `ConfigMap` + `ClickHouseInstallation` kubernetes manifest +Create the following `ConfigMap` + `ClickHouseInstallation` kubernetes manifest: + ```yaml --- apiVersion: v1 @@ -624,17 +630,18 @@ spec: containerPort: 7171 ``` -## How do incremental backups work to remote storage -- Incremental backup calculate increment only during execute `upload` or `create_remote` command or similar REST API request. -- When `use_embedded_backup_restore: false`, then incremental backup calculate increment only on table parts level, else increment backups also calculates based on `checksums.txt` for 23.3+ clickhouse version, look to ClickHouse documentation to fill the difference between [data parts](https://clickhouse.tech/docs/en/operations/system-tables/parts/) and [table partitions](https://clickhouse.tech/docs/en/operations/system-tables/partitions/). -- To calculate increment, backup which listed on `--diff-from` parameter is required to be present as local backup, look to `clickhouse-backup list` command results for ensure. -- Currently, during execute `clickhouse-backup upload --diff-from=base_backup` don't check `base_backup` exits on remote storage, be careful. -- During upload operation `base_backup` added to current backup metadata as required. All data parts which exists in `base_backup` also mark in backup metadata table level with `required` flag and skip data uploading. -- During download, if backup contains link to `required` backup it will try to fully download first. This action apply recursively. If you have a chain of incremental backups, all incremental backups in the chain and first "full" will download to local storage. -- Size of increment depends not only on the intensity your data ingestion and also depends on the intensity background merges for data parts in your tables. Please increase how much rows you will ingest during one INSERT query and don't apply often [table data mutations](https://clickhouse.tech/docs/en/operations/system-tables/mutations/). -- Look to [ClickHouse documentation](https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/) and try to understand how exactly `*MergeTree` table engine works. +## How incremental backups work with remote storage +- Incremental backup calculates the increment only while executing `upload` or `create_remote` commands or similar REST API requests. +- When `use_embedded_backup_restore: false`, then incremental backup calculates the increment only on the table parts level. +- When `use_embedded_backup_restore: true`, then incremental backup calculates by the checksums on file level, this approach more effective. +- For ClickHouse version 23.3+, see the ClickHouse documentation to find the difference between [data parts](https://clickhouse.com/docs/en/operations/system-tables/parts/) and [table partitions](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key). +- To calculate the increment, the backup listed on the `--diff-from` parameter is required to be present as a local backup. Check the `clickhouse-backup list` command results for errors. +- During upload, `base_backup` is added to current backup metadata as `required_backup` in `backup_name/metadata.json`. All data parts that exist in `base_backup` also mark in the backup metadata table level with `required` flag in `backup_name/metadata/database/table.json` and skip data uploading. +- During download, if a backup contains link to a `required_backup`, each table which contains parts marked as `required` will download these parts to local storage after complete downloading for non `required` parts. If you have a chain of incremental backups and required parts exist in this chain, then this action applies recursively. +- The size of the increment depends not only on the intensity of your data ingestion but also on the intensity of background merges for data parts in your tables. Please increase how many rows you will ingest during one INSERT query and don't do frequent [table data mutations](https://clickhouse.com/docs/en/operations/system-tables/mutations/). +- See the [ClickHouse documentation](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/) for information on how the `*MergeTree` table engine works. -## How to work `watch` command -Current implementation simple and will improve in next releases -- When `watch` command start, it call create_remote+delete command sequence to make `full` backup -- Then it wait `watch-interval` time period and call create_remote+delete command sequence again, type of backup will `full` if `full-interval` expired after last full backup created and `incremental`, if not. +## How to watch backups work +The current implementation is simple and will improve in next releases. +- When the `watch` command starts, it calls the `create_remote+delete command` sequence to make a `full` backup +- Then it waits `watch-interval` time period and calls the `create_remote+delete` command sequence again. The type of backup will be `full` if `full-interval` expired after last full backup created and `incremental` if not. diff --git a/Makefile b/Makefile index 15368c36..52d51bbf 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,8 @@ define DESC = Support of incremental backups on remote storages' endef GO_BUILD = go build -buildvcs=false -ldflags "-X 'main.version=$(VERSION)' -X 'main.gitCommit=$(GIT_COMMIT)' -X 'main.buildDate=$(DATE)'" -GO_BUILD_STATIC = go build -buildvcs=false -ldflags "-X 'main.version=$(VERSION)-fips' -X 'main.gitCommit=$(GIT_COMMIT)' -X 'main.buildDate=$(DATE)' -linkmode=external -extldflags '-static'" +GO_BUILD_STATIC = go build -buildvcs=false -ldflags "-X 'main.version=$(VERSION)' -X 'main.gitCommit=$(GIT_COMMIT)' -X 'main.buildDate=$(DATE)' -linkmode=external -extldflags '-static'" +GO_BUILD_STATIC_FIPS = go build -buildvcs=false -ldflags "-X 'main.version=$(VERSION)-fips' -X 'main.gitCommit=$(GIT_COMMIT)' -X 'main.buildDate=$(DATE)' -linkmode=external -extldflags '-static'" PKG_FILES = build/$(NAME)_$(VERSION).amd64.deb build/$(NAME)_$(VERSION).arm64.deb build/$(NAME)-$(VERSION)-1.amd64.rpm build/$(NAME)-$(VERSION)-1.arm64.rpm HOST_OS = $(shell bash -c 'source <(go env) && echo $$GOHOSTOS') HOST_ARCH = $(shell bash -c 'source <(go env) && echo $$GOHOSTARCH') @@ -55,7 +56,7 @@ build/linux/arm64/$(NAME)-fips build/darwin/arm64/$(NAME)-fips: GOARCH = arm64 build/linux/amd64/$(NAME)-fips build/linux/arm64/$(NAME)-fips: GOOS = linux build/darwin/amd64/$(NAME)-fips build/darwin/arm64/$(NAME)-fips: GOOS = darwin build/linux/amd64/$(NAME)-fips build/darwin/amd64/$(NAME)-fips: - CC=musl-gcc GOEXPERIMENT=boringcrypto CGO_ENABLED=1 GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO_BUILD_STATIC) -o $@ ./cmd/$(NAME) && \ + CC=musl-gcc GOEXPERIMENT=boringcrypto CGO_ENABLED=1 GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO_BUILD_STATIC_FIPS) -o $@ ./cmd/$(NAME) && \ go tool nm $@ > /tmp/$(NAME)-fips-tags.txt && \ grep '_Cfunc__goboringcrypto_' /tmp/$(NAME)-fips-tags.txt 1> /dev/null && \ rm -fv /tmp/$(NAME)-fips-tags.txt @@ -63,7 +64,7 @@ build/linux/amd64/$(NAME)-fips build/darwin/amd64/$(NAME)-fips: # TODO remove ugly workaround, https://www.perplexity.ai/search/2ead4c04-060a-4d78-a75f-f26835238438 build/linux/arm64/$(NAME)-fips build/darwin/arm64/$(NAME)-fips: bash -xce 'if [[ ! -f ~/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc ]]; then wget -q -P ~ https://musl.cc/aarch64-linux-musl-cross.tgz; tar -xvf ~/aarch64-linux-musl-cross.tgz -C ~; fi' && \ - CC=~/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc GOEXPERIMENT=boringcrypto CGO_ENABLED=1 GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO_BUILD_STATIC) -o $@ ./cmd/$(NAME) && \ + CC=~/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc GOEXPERIMENT=boringcrypto CGO_ENABLED=1 GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO_BUILD_STATIC_FIPS) -o $@ ./cmd/$(NAME) && \ go tool nm $@ > /tmp/$(NAME)-fips-tags.txt && \ grep '_Cfunc__goboringcrypto_' /tmp/$(NAME)-fips-tags.txt 1> /dev/null && \ rm -fv /tmp/$(NAME)-fips-tags.txt @@ -128,36 +129,40 @@ $(NAME)/$(NAME)-race: build-race-fips: $(NAME)/$(NAME)-race-fips $(NAME)/$(NAME)-race-fips: - CC=musl-gcc GOEXPERIMENT=boringcrypto CGO_ENABLED=1 $(GO_BUILD_STATIC) -cover -gcflags "all=-N -l" -race -o $@ ./cmd/$(NAME) + CC=musl-gcc GOEXPERIMENT=boringcrypto CGO_ENABLED=1 $(GO_BUILD_STATIC_FIPS) -cover -gcflags "all=-N -l" -race -o $@ ./cmd/$(NAME) # run `docker buildx create --use` first time build-race-docker: - bash -xce 'docker buildx build --build-arg CLICKHOUSE_VERSION=$${CLICKHOUSE_VERSION:-latest} --build-arg CLICKHOUSE_IMAGE=$${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server} --tag $(NAME):build-race --target make-build-race --progress plain --load . && \ + bash -xce 'docker buildx build --build-arg CLICKHOUSE_VERSION=$${CLICKHOUSE_VERSION:-latest} --build-arg CLICKHOUSE_IMAGE=$${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server} --build-arg VERSION=$(VERSION) \ + --tag $(NAME):build-race --target make-build-race --progress plain --load . && \ mkdir -pv ./$(NAME) && \ DOCKER_ID=$$(docker create $(NAME):build-race) && \ - docker cp -q $${DOCKER_ID}:/src/$(NAME)/$(NAME)-race ./$(NAME)/ && \ + docker cp $${DOCKER_ID}:/src/$(NAME)/$(NAME)-race ./$(NAME)/ && \ docker rm -f "$${DOCKER_ID}" && \ cp -fl ./$(NAME)/$(NAME)-race ./$(NAME)/$(NAME)-race-docker' build-race-fips-docker: - bash -xce 'docker buildx build --build-arg CLICKHOUSE_VERSION=$${CLICKHOUSE_VERSION:-latest} --build-arg CLICKHOUSE_IMAGE=$${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server} --tag $(NAME):build-race-fips --target make-build-race-fips --progress plain --load . && \ + bash -xce 'docker buildx build --build-arg CLICKHOUSE_VERSION=$${CLICKHOUSE_VERSION:-latest} --build-arg CLICKHOUSE_IMAGE=$${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server} --build-arg VERSION=$(VERSION) \ + --tag $(NAME):build-race-fips --target make-build-race-fips --progress plain --load . && \ mkdir -pv ./$(NAME) && \ DOCKER_ID=$$(docker create $(NAME):build-race-fips) && \ - docker cp -q $${DOCKER_ID}:/src/$(NAME)/$(NAME)-race-fips ./$(NAME)/ && \ + docker cp $${DOCKER_ID}:/src/$(NAME)/$(NAME)-race-fips ./$(NAME)/ && \ docker rm -f "$${DOCKER_ID}" && \ cp -fl ./$(NAME)/$(NAME)-race-fips ./$(NAME)/$(NAME)-race-fips-docker' build-docker: - bash -xce 'docker buildx build --build-arg CLICKHOUSE_VERSION=$${CLICKHOUSE_VERSION:-latest} --build-arg CLICKHOUSE_IMAGE=$${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server} --tag $(NAME):build-docker --target make-build-docker --progress plain --load . && \ + bash -xce 'docker buildx build --build-arg CLICKHOUSE_VERSION=$${CLICKHOUSE_VERSION:-latest} --build-arg CLICKHOUSE_IMAGE=$${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server} --build-arg VERSION=$(VERSION) \ + --tag $(NAME):build-docker --target make-build-docker --progress plain --load . && \ mkdir -pv ./build && \ DOCKER_ID=$$(docker create $(NAME):build-docker) && \ - docker cp -q $${DOCKER_ID}:/src/build/ ./build/ && \ + docker cp $${DOCKER_ID}:/src/build/ ./build/ && \ docker rm -f "$${DOCKER_ID}"' build-fips-docker: - bash -xce 'docker buildx build --build-arg CLICKHOUSE_VERSION=$${CLICKHOUSE_VERSION:-latest} --build-arg CLICKHOUSE_IMAGE=$${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server} --tag $(NAME):build-docker-fips --target make-build-fips --progress plain --load . && \ + bash -xce 'docker buildx build --build-arg CLICKHOUSE_VERSION=$${CLICKHOUSE_VERSION:-latest} --build-arg CLICKHOUSE_IMAGE=$${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server} --build-arg VERSION=$(VERSION) \ + --tag $(NAME):build-docker-fips --target make-build-fips --progress plain --load . && \ mkdir -pv ./build && \ DOCKER_ID=$$(docker create $(NAME):build-docker) && \ - docker cp -q $${DOCKER_ID}:/src/build/ ./build/ && \ + docker cp $${DOCKER_ID}:/src/build/ ./build/ && \ docker rm -f "$${DOCKER_ID}"' diff --git a/Manual.md b/Manual.md index 7b38e03a..34fb89bb 100644 --- a/Manual.md +++ b/Manual.md @@ -4,12 +4,14 @@ NAME: clickhouse-backup tables - List of tables, exclude skip_tables USAGE: - clickhouse-backup tables [-t, --tables=.]] [--all] + clickhouse-backup tables [--tables=.
] [--remote-backup=] [--all] OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --all, -a print table even when match with skip_tables pattern - --table value, --tables value, -t value list tables only match with table name patterns, separated by comma, allow ? and * as wildcard + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --all, -a Print table even when match with skip_tables pattern + --table value, --tables value, -t value List tables only match with table name patterns, separated by comma, allow ? and * as wildcard + --remote-backup value List tables from remote backup ``` ### CLI command - create @@ -24,18 +26,23 @@ DESCRIPTION: Create new backup OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --table value, --tables value, -t value create backup only matched with table name patterns, separated by comma, allow ? and * as wildcard - --partitions partition_id create backup only for selected partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ - --schema, -s Backup schemas only + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --table value, --tables value, -t value Create backup only matched with table name patterns, separated by comma, allow ? and * as wildcard + --diff-from-remote value Create incremental embedded backup or upload incremental object disk data based on other remote backup name + --partitions partition_id Create backup only for selected partition names, separated by comma +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ + --schema, -s Backup schemas only, will skip data --rbac, --backup-rbac, --do-backup-rbac Backup RBAC related objects --configs, --backup-configs, --do-backup-configs Backup 'clickhouse-server' configuration files - --skip-check-parts-columns skip check system.parts_columns to disallow backup inconsistent column types for data parts + --rbac-only Backup RBAC related objects only, will skip backup data, will backup schema only if --schema added + --configs-only Backup 'clickhouse-server' configuration files only, will skip backup data, will backup schema only if --schema added + --skip-check-parts-columns Skip check system.parts_columns to disallow backup inconsistent column types for data parts ``` ### CLI command - create_remote @@ -50,21 +57,26 @@ DESCRIPTION: Create and upload OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --table value, --tables value, -t value create and upload backup only matched with table name patterns, separated by comma, allow ? and * as wildcard - --partitions partition_id create and upload backup only for selected partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ - --diff-from value local backup name which used to upload current backup as incremental - --diff-from-remote value remote backup name which used to upload current backup as incremental - --schema, -s Backup and upload metadata schema only + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --table value, --tables value, -t value Create and upload backup only matched with table name patterns, separated by comma, allow ? and * as wildcard + --partitions partition_id Create and upload backup only for selected partition names, separated by comma +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ + --diff-from value Local backup name which used to upload current backup as incremental + --diff-from-remote value Remote backup name which used to upload current backup as incremental + --schema, -s Backup and upload metadata schema only, will skip data backup --rbac, --backup-rbac, --do-backup-rbac Backup and upload RBAC related objects --configs, --backup-configs, --do-backup-configs Backup and upload 'clickhouse-server' configuration files + --rbac-only Backup RBAC related objects only, will skip backup data, will backup schema only if --schema added + --configs-only Backup 'clickhouse-server' configuration files only, will skip backup data, will backup schema only if --schema added --resume, --resumable Save intermediate upload state and resume upload if backup exists on remote storage, ignore when 'remote_storage: custom' or 'use_embedded_backup_restore: true' - --skip-check-parts-columns skip check system.parts_columns to disallow backup inconsistent column types for data parts + --skip-check-parts-columns Skip check system.parts_columns to disallow backup inconsistent column types for data parts + --delete, --delete-source, --delete-local explicitly delete local backup during upload ``` ### CLI command - upload @@ -76,18 +88,21 @@ USAGE: clickhouse-backup upload [-t, --tables=.
] [--partitions=] [-s, --schema] [--diff-from=] [--diff-from-remote=] [--resumable] OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --diff-from value local backup name which used to upload current backup as incremental - --diff-from-remote value remote backup name which used to upload current backup as incremental - --table value, --tables value, -t value Upload data only for matched table name patterns, separated by comma, allow ? and * as wildcard - --partitions partition_id Upload backup only for selected partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ - --schema, -s Upload schemas only - --resume, --resumable Save intermediate upload state and resume upload if backup exists on remote storage, ignored with 'remote_storage: custom' or 'use_embedded_backup_restore: true' + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --diff-from value Local backup name which used to upload current backup as incremental + --diff-from-remote value Remote backup name which used to upload current backup as incremental + --table value, --tables value, -t value Upload data only for matched table name patterns, separated by comma, allow ? and * as wildcard + --partitions partition_id Upload backup only for selected partition names, separated by comma +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ + --schema, -s Upload schemas only + --resume, --resumable Save intermediate upload state and resume upload if backup exists on remote storage, ignored with 'remote_storage: custom' or 'use_embedded_backup_restore: true' + --delete, --delete-source, --delete-local explicitly delete local backup during upload ``` ### CLI command - list @@ -99,7 +114,8 @@ USAGE: clickhouse-backup list [all|local|remote] [latest|previous] OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter ``` ### CLI command - download @@ -111,14 +127,16 @@ USAGE: clickhouse-backup download [-t, --tables=.
] [--partitions=] [-s, --schema] [--resumable] OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --table value, --tables value, -t value Download objects which matched with table name patterns, separated by comma, allow ? and * as wildcard - --partitions partition_id Download backup data only for selected partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --table value, --tables value, -t value Download objects which matched with table name patterns, separated by comma, allow ? and * as wildcard + --partitions partition_id Download backup data only for selected partition names, separated by comma +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ --schema, -s Download schema only --resume, --resumable Save intermediate download state and resume download if backup exists on local storage, ignored with 'remote_storage: custom' or 'use_embedded_backup_restore: true' @@ -129,24 +147,29 @@ NAME: clickhouse-backup restore - Create schema and restore data from backup USAGE: - clickhouse-backup restore [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--partitions=] [-s, --schema] [-d, --data] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] + clickhouse-backup restore [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--tm, --restore-table-mapping=:[,<...>]] [--partitions=] [-s, --schema] [-d, --data] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] OPTIONS: --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter --table value, --tables value, -t value Restore only database and objects which matched with table name patterns, separated by comma, allow ? and * as wildcard --restore-database-mapping value, -m value Define the rule to restore data. For the database not defined in this struct, the program will not deal with it. + --restore-table-mapping value, --tm value Define the rule to restore data. For the table not defined in this struct, the program will not deal with it. --partitions partition_id Restore backup only for selected partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ --schema, -s Restore schema only --data, -d Restore data only --rm, --drop Drop exists schema objects before restore -i, --ignore-dependencies Ignore dependencies when drop exists schema objects --rbac, --restore-rbac, --do-restore-rbac Restore RBAC related objects --configs, --restore-configs, --do-restore-configs Restore 'clickhouse-server' CONFIG related files + --rbac-only Restore RBAC related objects only, will skip backup data, will backup schema only if --schema added + --configs-only Restore 'clickhouse-server' configuration files only, will skip backup data, will backup schema only if --schema added ``` ### CLI command - restore_remote @@ -155,24 +178,29 @@ NAME: clickhouse-backup restore_remote - Download and restore USAGE: - clickhouse-backup restore_remote [--schema] [--data] [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--partitions=] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] [--skip-rbac] [--skip-configs] [--resumable] + clickhouse-backup restore_remote [--schema] [--data] [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--tm, --restore-table-mapping=:[,<...>]] [--partitions=] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] [--skip-rbac] [--skip-configs] [--resumable] OPTIONS: --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter --table value, --tables value, -t value Download and restore objects which matched with table name patterns, separated by comma, allow ? and * as wildcard --restore-database-mapping value, -m value Define the rule to restore data. For the database not defined in this struct, the program will not deal with it. + --restore-table-mapping value, --tm value Define the rule to restore data. For the table not defined in this struct, the program will not deal with it. --partitions partition_id Download and restore backup only for selected partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ --schema, -s Download and Restore schema only --data, -d Download and Restore data only --rm, --drop Drop schema objects before restore -i, --ignore-dependencies Ignore dependencies when drop exists schema objects --rbac, --restore-rbac, --do-restore-rbac Download and Restore RBAC related objects --configs, --restore-configs, --do-restore-configs Download and Restore 'clickhouse-server' CONFIG related files + --rbac-only Restore RBAC related objects only, will skip backup data, will backup schema only if --schema added + --configs-only Restore 'clickhouse-server' configuration files only, will skip backup data, will backup schema only if --schema added --resume, --resumable Save intermediate upload state and resume upload if backup exists on remote storage, ignored with 'remote_storage: custom' or 'use_embedded_backup_restore: true' ``` @@ -185,7 +213,8 @@ USAGE: clickhouse-backup delete OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter ``` ### CLI command - default-config @@ -197,7 +226,8 @@ USAGE: clickhouse-backup default-config [command options] [arguments...] OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter ``` ### CLI command - print-config @@ -209,7 +239,8 @@ USAGE: clickhouse-backup print-config [command options] [arguments...] OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter ``` ### CLI command - clean @@ -221,7 +252,8 @@ USAGE: clickhouse-backup clean [command options] [arguments...] OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter ``` ### CLI command - clean_remote_broken @@ -233,7 +265,8 @@ USAGE: clickhouse-backup clean_remote_broken [command options] [arguments...] OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter ``` ### CLI command - watch @@ -248,21 +281,23 @@ DESCRIPTION: Execute create_remote + delete local, create full backup every `--full-interval`, create and upload incremental backup every `--watch-interval` use previous backup as base with `--diff-from-remote` option, use `backups_to_keep_remote` config option for properly deletion remote backups, will delete old backups which not have references from other backups OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --watch-interval value Interval for run 'create_remote' + 'delete local' for incremental backup, look format https://pkg.go.dev/time#ParseDuration - --full-interval value Interval for run 'create_remote'+'delete local' when stop create incremental backup sequence and create full backup, look format https://pkg.go.dev/time#ParseDuration - --watch-backup-name-template value Template for new backup name, could contain names from system.macros, {type} - full or incremental and {time:LAYOUT}, look to https://go.dev/src/time/format.go for layout examples - --table value, --tables value, -t value Create and upload only objects which matched with table name patterns, separated by comma, allow ? and * as wildcard - --partitions partition_id partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --watch-interval value Interval for run 'create_remote' + 'delete local' for incremental backup, look format https://pkg.go.dev/time#ParseDuration + --full-interval value Interval for run 'create_remote'+'delete local' when stop create incremental backup sequence and create full backup, look format https://pkg.go.dev/time#ParseDuration + --watch-backup-name-template value Template for new backup name, could contain names from system.macros, {type} - full or incremental and {time:LAYOUT}, look to https://go.dev/src/time/format.go for layout examples + --table value, --tables value, -t value Create and upload only objects which matched with table name patterns, separated by comma, allow ? and * as wildcard + --partitions partition_id Partitions names, separated by comma +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ --schema, -s Schemas only --rbac, --backup-rbac, --do-backup-rbac Backup RBAC related objects only --configs, --backup-configs, --do-backup-configs Backup `clickhouse-server' configuration files only - --skip-check-parts-columns skip check system.parts_columns to disallow backup inconsistent column types for data parts + --skip-check-parts-columns Skip check system.parts_columns to disallow backup inconsistent column types for data parts ``` ### CLI command - server @@ -274,10 +309,11 @@ USAGE: clickhouse-backup server [command options] [arguments...] OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --watch run watch go-routine for 'create_remote' + 'delete local', after API server startup - --watch-interval value Interval for run 'create_remote' + 'delete local' for incremental backup, look format https://pkg.go.dev/time#ParseDuration - --full-interval value Interval for run 'create_remote'+'delete local' when stop create incremental backup sequence and create full backup, look format https://pkg.go.dev/time#ParseDuration - --watch-backup-name-template value Template for new backup name, could contain names from system.macros, {type} - full or incremental and {time:LAYOUT}, look to https://go.dev/src/time/format.go for layout examples + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --watch Run watch go-routine for 'create_remote' + 'delete local', after API server startup + --watch-interval value Interval for run 'create_remote' + 'delete local' for incremental backup, look format https://pkg.go.dev/time#ParseDuration + --full-interval value Interval for run 'create_remote'+'delete local' when stop create incremental backup sequence and create full backup, look format https://pkg.go.dev/time#ParseDuration + --watch-backup-name-template value Template for new backup name, could contain names from system.macros, {type} - full or incremental and {time:LAYOUT}, look to https://go.dev/src/time/format.go for layout examples ``` diff --git a/ReadMe.md b/ReadMe.md index fd4ea015..fdc38fff 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -1,16 +1,19 @@ -# clickhouse-backup +# Altinity Backup for ClickHouse® [![Build](https://github.com/Altinity/clickhouse-backup/actions/workflows/build.yaml/badge.svg?branch=master)](https://github.com/Altinity/clickhouse-backup/actions/workflows/build.yaml) [![GoDoc](https://godoc.org/github.com/Altinity/clickhouse-backup?status.svg)](http://godoc.org/github.com/Altinity/clickhouse-backup) [![Telegram](https://img.shields.io/badge/telegram-join%20chat-3796cd.svg)](https://t.me/clickhousebackup) [![Docker Image](https://img.shields.io/docker/pulls/altinity/clickhouse-backup.svg)](https://hub.docker.com/r/altinity/clickhouse-backup) +[![Downloads](https://img.shields.io/github/downloads/Altinity/clickhouse-backup/total.svg)](http://github.com/Altinity/clickhouse-backup/releases) [![Coverage Status](https://coveralls.io/repos/github/Altinity/clickhouse-backup/badge.svg)](https://coveralls.io/github/Altinity/clickhouse-backup) -A tool for easy ClickHouse backup and restore with support for many cloud and non-cloud storage types. -To make data backup `clickhouse-backup` requires access to the same files as `clickhouse-server` in `/var/lib/clickhouse` folders. -So prefer running `clickhouse-backup` on the same host or same Kubernetes Pod or the neighbor container on the same host where `clickhouse-server` ran. -You can backup only schema from remote hosts. +A tool for easy backup and restore utility for ClickHouse databases with support for many cloud and non-cloud storage types. + +### Don't run `clickhouse-backup` remotely +To backup data, `clickhouse-backup` requires access to the same files as `clickhouse-server` in `/var/lib/clickhouse` folders. +For that reason, it's required to run `clickhouse-backup` on the same host or same Kubernetes Pod or the neighbor container on the same host where `clickhouse-server` ran. +**WARNING** You can backup only schema when connect to remote `clickhouse-server` hosts. ## Features @@ -18,16 +21,28 @@ You can backup only schema from remote hosts. - Efficient storing of multiple backups on the file system - Uploading and downloading with streaming compression - Works with AWS, GCS, Azure, Tencent COS, FTP, SFTP -- **Support of Atomic Database Engine** -- **Support of multi disks installations** +- **Support for Atomic Database Engine** +- **Support for multi disks installations** - **Support for custom remote storage types via `rclone`, `kopia`, `restic`, `rsync` etc** -- Support of incremental backups on remote storage +- Support for incremental backups on remote storage ## Limitations - ClickHouse above 1.1.54394 is supported - Only MergeTree family tables engines (more table types for `clickhouse-server` 22.7+ and `USE_EMBEDDED_BACKUP_RESTORE=true`) +## Support + +Altinity is the primary maintainer of clickhouse-backup. We offer a range of software and +services related to ClickHouse. + +- [Official website](https://altinity.com/) - Get a high level overview of Altinity and our offerings. +- [Altinity.Cloud](https://altinity.com/cloud-database/) - Run ClickHouse in our cloud or yours. +- [Altinity Support](https://altinity.com/support/) - Get Enterprise-class support for ClickHouse. +- [Slack](https://altinitydbworkspace.slack.com/join/shared_invite/zt-1togw9b4g-N0ZOXQyEyPCBh_7IEHUjdw#/shared-invite/email) - Talk directly with ClickHouse users and Altinity devs. +- [Contact us](https://hubs.la/Q020sH3Z0) - Contact Altinity with your questions or issues. +- [Free consultation](https://hubs.la/Q020sHkv0) - Get a free consultation with a ClickHouse expert today. + ## Installation Download the latest binary from the [releases](https://github.com/Altinity/clickhouse-backup/releases) page and decompress with: @@ -36,7 +51,7 @@ Download the latest binary from the [releases](https://github.com/Altinity/click tar -zxvf clickhouse-backup.tar.gz ``` -Use the official tiny Docker image and run it on host with `clickhouse-server` installed: +Use the official tiny Docker image and run it on a host with `clickhouse-server` installed: ```shell docker run -u $(id -u clickhouse) --rm -it --network host -v "/var/lib/clickhouse:/var/lib/clickhouse" \ @@ -47,321 +62,52 @@ docker run -u $(id -u clickhouse) --rm -it --network host -v "/var/lib/clickhous altinity/clickhouse-backup --help ``` -Build from the sources: +Build from the sources (required go 1.21+): ```shell -GO111MODULE=on go get github.com/Altinity/clickhouse-backup/cmd/clickhouse-backup -``` - -## Common CLI Usage - -### CLI command - tables -``` -NAME: - clickhouse-backup tables - List of tables, exclude skip_tables - -USAGE: - clickhouse-backup tables [-t, --tables=.
]] [--all] - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --all, -a print table even when match with skip_tables pattern - --table value, --tables value, -t value list tables only match with table name patterns, separated by comma, allow ? and * as wildcard - -``` -### CLI command - create -``` -NAME: - clickhouse-backup create - Create new backup - -USAGE: - clickhouse-backup create [-t, --tables=.
] [--partitions=] [-s, --schema] [--rbac] [--configs] [--skip-check-parts-columns] - -DESCRIPTION: - Create new backup - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --table value, --tables value, -t value create backup only matched with table name patterns, separated by comma, allow ? and * as wildcard - --partitions partition_id create backup only for selected partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ - --schema, -s Backup schemas only - --rbac, --backup-rbac, --do-backup-rbac Backup RBAC related objects - --configs, --backup-configs, --do-backup-configs Backup 'clickhouse-server' configuration files - --skip-check-parts-columns skip check system.parts_columns to disallow backup inconsistent column types for data parts - -``` -### CLI command - create_remote -``` -NAME: - clickhouse-backup create_remote - Create and upload new backup - -USAGE: - clickhouse-backup create_remote [-t, --tables=.
] [--partitions=] [--diff-from=] [--diff-from-remote=] [--schema] [--rbac] [--configs] [--resumable] [--skip-check-parts-columns] - -DESCRIPTION: - Create and upload - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --table value, --tables value, -t value create and upload backup only matched with table name patterns, separated by comma, allow ? and * as wildcard - --partitions partition_id create and upload backup only for selected partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ - --diff-from value local backup name which used to upload current backup as incremental - --diff-from-remote value remote backup name which used to upload current backup as incremental - --schema, -s Backup and upload metadata schema only - --rbac, --backup-rbac, --do-backup-rbac Backup and upload RBAC related objects - --configs, --backup-configs, --do-backup-configs Backup and upload 'clickhouse-server' configuration files - --resume, --resumable Save intermediate upload state and resume upload if backup exists on remote storage, ignore when 'remote_storage: custom' or 'use_embedded_backup_restore: true' - --skip-check-parts-columns skip check system.parts_columns to disallow backup inconsistent column types for data parts - -``` -### CLI command - upload -``` -NAME: - clickhouse-backup upload - Upload backup to remote storage - -USAGE: - clickhouse-backup upload [-t, --tables=.
] [--partitions=] [-s, --schema] [--diff-from=] [--diff-from-remote=] [--resumable] - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --diff-from value local backup name which used to upload current backup as incremental - --diff-from-remote value remote backup name which used to upload current backup as incremental - --table value, --tables value, -t value Upload data only for matched table name patterns, separated by comma, allow ? and * as wildcard - --partitions partition_id Upload backup only for selected partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ - --schema, -s Upload schemas only - --resume, --resumable Save intermediate upload state and resume upload if backup exists on remote storage, ignored with 'remote_storage: custom' or 'use_embedded_backup_restore: true' - -``` -### CLI command - list -``` -NAME: - clickhouse-backup list - List of backups - -USAGE: - clickhouse-backup list [all|local|remote] [latest|previous] - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - -``` -### CLI command - download +GO111MODULE=on go install github.com/Altinity/clickhouse-backup/v2/cmd/clickhouse-backup@latest ``` -NAME: - clickhouse-backup download - Download backup from remote storage - -USAGE: - clickhouse-backup download [-t, --tables=.
] [--partitions=] [-s, --schema] [--resumable] - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --table value, --tables value, -t value Download objects which matched with table name patterns, separated by comma, allow ? and * as wildcard - --partitions partition_id Download backup data only for selected partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ - --schema, -s Download schema only - --resume, --resumable Save intermediate download state and resume download if backup exists on local storage, ignored with 'remote_storage: custom' or 'use_embedded_backup_restore: true' - -``` -### CLI command - restore -``` -NAME: - clickhouse-backup restore - Create schema and restore data from backup - -USAGE: - clickhouse-backup restore [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--partitions=] [-s, --schema] [-d, --data] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --table value, --tables value, -t value Restore only database and objects which matched with table name patterns, separated by comma, allow ? and * as wildcard - --restore-database-mapping value, -m value Define the rule to restore data. For the database not defined in this struct, the program will not deal with it. - --partitions partition_id Restore backup only for selected partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ - --schema, -s Restore schema only - --data, -d Restore data only - --rm, --drop Drop exists schema objects before restore - -i, --ignore-dependencies Ignore dependencies when drop exists schema objects - --rbac, --restore-rbac, --do-restore-rbac Restore RBAC related objects - --configs, --restore-configs, --do-restore-configs Restore 'clickhouse-server' CONFIG related files - -``` -### CLI command - restore_remote -``` -NAME: - clickhouse-backup restore_remote - Download and restore - -USAGE: - clickhouse-backup restore_remote [--schema] [--data] [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--partitions=] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] [--skip-rbac] [--skip-configs] [--resumable] - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --table value, --tables value, -t value Download and restore objects which matched with table name patterns, separated by comma, allow ? and * as wildcard - --restore-database-mapping value, -m value Define the rule to restore data. For the database not defined in this struct, the program will not deal with it. - --partitions partition_id Download and restore backup only for selected partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ - --schema, -s Download and Restore schema only - --data, -d Download and Restore data only - --rm, --drop Drop schema objects before restore - -i, --ignore-dependencies Ignore dependencies when drop exists schema objects - --rbac, --restore-rbac, --do-restore-rbac Download and Restore RBAC related objects - --configs, --restore-configs, --do-restore-configs Download and Restore 'clickhouse-server' CONFIG related files - --resume, --resumable Save intermediate upload state and resume upload if backup exists on remote storage, ignored with 'remote_storage: custom' or 'use_embedded_backup_restore: true' - -``` -### CLI command - delete -``` -NAME: - clickhouse-backup delete - Delete specific backup - -USAGE: - clickhouse-backup delete - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - -``` -### CLI command - default-config -``` -NAME: - clickhouse-backup default-config - Print default config - -USAGE: - clickhouse-backup default-config [command options] [arguments...] - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - -``` -### CLI command - print-config -``` -NAME: - clickhouse-backup print-config - Print current config merged with environment variables - -USAGE: - clickhouse-backup print-config [command options] [arguments...] - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - -``` -### CLI command - clean -``` -NAME: - clickhouse-backup clean - Remove data in 'shadow' folder from all 'path' folders available from 'system.disks' - -USAGE: - clickhouse-backup clean [command options] [arguments...] - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - -``` -### CLI command - clean_remote_broken -``` -NAME: - clickhouse-backup clean_remote_broken - Remove all broken remote backups - -USAGE: - clickhouse-backup clean_remote_broken [command options] [arguments...] - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - -``` -### CLI command - watch -``` -NAME: - clickhouse-backup watch - Run infinite loop which create full + incremental backup sequence to allow efficient backup sequences - -USAGE: - clickhouse-backup watch [--watch-interval=1h] [--full-interval=24h] [--watch-backup-name-template=shard{shard}-{type}-{time:20060102150405}] [-t, --tables=.
] [--partitions=] [--schema] [--rbac] [--configs] [--skip-check-parts-columns] - -DESCRIPTION: - Execute create_remote + delete local, create full backup every `--full-interval`, create and upload incremental backup every `--watch-interval` use previous backup as base with `--diff-from-remote` option, use `backups_to_keep_remote` config option for properly deletion remote backups, will delete old backups which not have references from other backups - -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --watch-interval value Interval for run 'create_remote' + 'delete local' for incremental backup, look format https://pkg.go.dev/time#ParseDuration - --full-interval value Interval for run 'create_remote'+'delete local' when stop create incremental backup sequence and create full backup, look format https://pkg.go.dev/time#ParseDuration - --watch-backup-name-template value Template for new backup name, could contain names from system.macros, {type} - full or incremental and {time:LAYOUT}, look to https://go.dev/src/time/format.go for layout examples - --table value, --tables value, -t value Create and upload only objects which matched with table name patterns, separated by comma, allow ? and * as wildcard - --partitions partition_id partition names, separated by comma -if PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format -if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format -if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format -values depends on field types in your table, use single quote for String and Date/DateTime related types -look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ - --schema, -s Schemas only - --rbac, --backup-rbac, --do-backup-rbac Backup RBAC related objects only - --configs, --backup-configs, --do-backup-configs Backup `clickhouse-server' configuration files only - --skip-check-parts-columns skip check system.parts_columns to disallow backup inconsistent column types for data parts - -``` -### CLI command - server -``` -NAME: - clickhouse-backup server - Run API server - -USAGE: - clickhouse-backup server [command options] [arguments...] -OPTIONS: - --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] - --watch run watch go-routine for 'create_remote' + 'delete local', after API server startup - --watch-interval value Interval for run 'create_remote' + 'delete local' for incremental backup, look format https://pkg.go.dev/time#ParseDuration - --full-interval value Interval for run 'create_remote'+'delete local' when stop create incremental backup sequence and create full backup, look format https://pkg.go.dev/time#ParseDuration - --watch-backup-name-template value Template for new backup name, could contain names from system.macros, {type} - full or incremental and {time:LAYOUT}, look to https://go.dev/src/time/format.go for layout examples +## Brief description how clickhouse-backup works -``` +Data files are immutable in the `clickhouse-server`. +During a backup operation, `clickhouse-backup` creates file system hard links to existing `clickhouse-server` data parts via executing the `ALTER TABLE ... FREEZE` query. +During the restore operation, `clickhouse-backup` copies the hard links to the `detached` folder and executes the `ALTER TABLE ... ATTACH PART` query for each data part and each table in the backup. +A more detailed description is available here: https://www.youtube.com/watch?v=megsNh9Q-dw ## Default Config -By default, the config file is located at `/etc/clickhouse-backup/config.yml`, but it can be redefined via `CLICKHOUSE_BACKUP_CONFIG` environment variable. +By default, the config file is located at `/etc/clickhouse-backup/config.yml`, but it can be redefined via the `CLICKHOUSE_BACKUP_CONFIG` environment variable or via `--config` command line parameter. All options can be overwritten via environment variables. +Use `clickhouse-backup default-config` to print default config. + +## Explain config parameters +following values is not default, it just explain which each config parameter actually means Use `clickhouse-backup print-config` to print current config. ```yaml general: - remote_storage: none # REMOTE_STORAGE, if `none` then `upload` and `download` command will fail - max_file_size: 1073741824 # MAX_FILE_SIZE, 1G by default, useless when upload_by_part is true, use for split data parts files by archives - disable_progress_bar: true # DISABLE_PROGRESS_BAR, show progress bar during upload and download, makes sense only when `upload_concurrency` and `download_concurrency` is 1 + remote_storage: none # REMOTE_STORAGE, choice from: `azblob`,`gcs`,`s3`, etc; if `none` then `upload` and `download` commands will fail. + max_file_size: 1073741824 # MAX_FILE_SIZE, 1G by default, useless when upload_by_part is true, use to split data parts files by archives backups_to_keep_local: 0 # BACKUPS_TO_KEEP_LOCAL, how many latest local backup should be kept, 0 means all created backups will be stored on local disk # -1 means backup will keep after `create` but will delete after `create_remote` command - # You shall run `clickhouse-backup delete local ` command to remove temporary backup files from the local disk + # You can run `clickhouse-backup delete local ` command to remove temporary backup files from the local disk backups_to_keep_remote: 0 # BACKUPS_TO_KEEP_REMOTE, how many latest backup should be kept on remote storage, 0 means all uploaded backups will be stored on remote storage. # If old backups are required for newer incremental backup then it won't be deleted. Be careful with long incremental backup sequences. log_level: info # LOG_LEVEL, a choice from `debug`, `info`, `warn`, `error` allow_empty_backups: false # ALLOW_EMPTY_BACKUPS - # concurrency means parallel tables and parallel parts inside tables - # for example 4 means max 4 parallel tables and 4 parallel parts inside one table, so equals 16 concurrent streams + # Concurrency means parallel tables and parallel parts inside tables + # For example, 4 means max 4 parallel tables and 4 parallel parts inside one table, so equals 16 concurrent streams download_concurrency: 1 # DOWNLOAD_CONCURRENCY, max 255, by default, the value is round(sqrt(AVAILABLE_CPU_CORES / 2)) upload_concurrency: 1 # UPLOAD_CONCURRENCY, max 255, by default, the value is round(sqrt(AVAILABLE_CPU_CORES / 2)) - + + # Throttling speed for upload and download, calculates on part level, not the socket level, it means short period for high traffic values and then time to sleep + download_max_bytes_per_second: 0 # DOWNLOAD_MAX_BYTES_PER_SECOND, 0 means no throttling + upload_max_bytes_per_second: 0 # UPLOAD_MAX_BYTES_PER_SECOND, 0 means no throttling + + # when table data contains in system.disks with type=ObjectStorage, then we need execute remote copy object in object storage service provider, this parameter can restrict how many files will copied in parallel for each table + object_disk_server_side_copy_concurrency: 32 + # RESTORE_SCHEMA_ON_CLUSTER, execute all schema related SQL queries with `ON CLUSTER` clause as Distributed DDL. # Check `system.clusters` table for the correct cluster name, also `system.macros` can be used. # This isn't applicable when `use_embedded_backup_restore: true` @@ -373,6 +119,11 @@ general: # RESTORE_DATABASE_MAPPING, restore rules from backup databases to target databases, which is useful when changing destination database, all atomic tables will be created with new UUIDs. # The format for this env variable is "src_db1:target_db1,src_db2:target_db2". For YAML please continue using map syntax restore_database_mapping: {} + + # RESTORE_TABLE_MAPPING, restore rules from backup tables to target tables, which is useful when changing destination tables. + # The format for this env variable is "src_table1:target_table1,src_table2:target_table2". For YAML please continue using map syntax + restore_table_mapping: {} + retries_on_failure: 3 # RETRIES_ON_FAILURE, how many times to retry after a failure during upload or download retries_pause: 30s # RETRIES_PAUSE, duration time to pause after each download or upload failure @@ -381,13 +132,27 @@ general: watch_backup_name_template: "shard{shard}-{type}-{time:20060102150405}" # WATCH_BACKUP_NAME_TEMPLATE, used only for `watch` command, macros values will apply from `system.macros` for time:XXX, look format in https://go.dev/src/time/format.go sharded_operation_mode: none # SHARDED_OPERATION_MODE, how different replicas will shard backing up data for tables. Options are: none (no sharding), table (table granularity), database (database granularity), first-replica (on the lexicographically sorted first active replica). If left empty, then the "none" option will be set as default. + + cpu_nice_priority: 15 # CPU niceness priority, to allow throttling CPU intensive operation, more details https://manpages.ubuntu.com/manpages/xenial/man1/nice.1.html + io_nice_priority: "idle" # IO niceness priority, to allow throttling DISK intensive operation, more details https://manpages.ubuntu.com/manpages/xenial/man1/ionice.1.html + + rbac_backup_always: true # always, backup RBAC objects + rbac_resolve_conflicts: "recreate" # action, when RBAC object with the same name already exists, allow "recreate", "ignore", "fail" values clickhouse: username: default # CLICKHOUSE_USERNAME password: "" # CLICKHOUSE_PASSWORD host: localhost # CLICKHOUSE_HOST, To make backup data `clickhouse-backup` requires access to the same file system as clickhouse-server, so `host` should localhost or address of another docker container on the same machine, or IP address bound to some network interface on the same host. port: 9000 # CLICKHOUSE_PORT, don't use 8123, clickhouse-backup doesn't support HTTP protocol - # CLICKHOUSE_DISK_MAPPING, use this mapping when your `system.disks` are different between the source and destination clusters during backup and restore process - # The format for this env variable is "disk_name1:disk_path1,disk_name2:disk_path2". For YAML please continue using map syntax + # CLICKHOUSE_DISK_MAPPING, use this mapping when your `system.disks` are different between the source and destination clusters during backup and restore process. + # The format for this env variable is "disk_name1:disk_path1,disk_name2:disk_path2". For YAML please continue using map syntax. + # If destination disk is different from source backup disk then you need to specify the destination disk in the config file: + + # disk_mapping: + # disk_destination: /var/lib/clickhouse/disks/destination + + # `disk_destination` needs to be referenced in backup (source config), and all names from this map (`disk:path`) shall exist in `system.disks` on destination server. + # During download of the backup from remote location (s3), if `name` is not present in `disk_mapping` (on the destination server config too) then `default` disk path will used for download. + # `disk_mapping` is used to understand during download where downloaded parts shall be unpacked (which disk) on destination server and where to search for data parts directories during restore. disk_mapping: {} # CLICKHOUSE_SKIP_TABLES, the list of tables (pattern are allowed) which are ignored during backup and restore process # The format for this env variable is "pattern1,pattern2,pattern3". For YAML please continue using list syntax @@ -415,13 +180,15 @@ clickhouse: # available prefixes # - sql: will execute SQL query # - exec: will execute command via shell - restart_command: "sql:SYSTEM SHUTDOWN" + restart_command: "exec:systemctl restart clickhouse-server" ignore_not_exists_error_during_freeze: true # CLICKHOUSE_IGNORE_NOT_EXISTS_ERROR_DURING_FREEZE, helps to avoid backup failures when running frequent CREATE / DROP tables and databases during backup, `clickhouse-backup` will ignore `code: 60` and `code: 81` errors during execution of `ALTER TABLE ... FREEZE` check_replicas_before_attach: true # CLICKHOUSE_CHECK_REPLICAS_BEFORE_ATTACH, helps avoiding concurrent ATTACH PART execution when restoring ReplicatedMergeTree tables use_embedded_backup_restore: false # CLICKHOUSE_USE_EMBEDDED_BACKUP_RESTORE, use BACKUP / RESTORE SQL statements instead of regular SQL queries to use features of modern ClickHouse server versions - backup_mutations: true # CLICKHOUSE_BACKUP_MUTATIONS, allow backup mutations from system.mutations WHERE is_done AND apply it during restore + embedded_backup_disk: "" # CLICKHOUSE_EMBEDDED_BACKUP_DISK - disk from system.disks which will use when `use_embedded_backup_restore: true` + backup_mutations: true # CLICKHOUSE_BACKUP_MUTATIONS, allow backup mutations from system.mutations WHERE is_done=0 and apply it during restore restore_as_attach: false # CLICKHOUSE_RESTORE_AS_ATTACH, allow restore tables which have inconsistent data parts structure and mutations in progress check_parts_columns: true # CLICKHOUSE_CHECK_PARTS_COLUMNS, check data types from system.parts_columns during create backup to guarantee mutation is complete + max_connections: 0 # CLICKHOUSE_MAX_CONNECTIONS, how many parallel connections could be opened during operations azblob: endpoint_suffix: "core.windows.net" # AZBLOB_ENDPOINT_SUFFIX account_name: "" # AZBLOB_ACCOUNT_NAME @@ -429,7 +196,7 @@ azblob: sas: "" # AZBLOB_SAS use_managed_identity: false # AZBLOB_USE_MANAGED_IDENTITY container: "" # AZBLOB_CONTAINER - path: "" # AZBLOB_PATH, `system.macros` values could be applied as {macro_name} + path: "" # AZBLOB_PATH, `system.macros` values can be applied as {macro_name} object_disk_path: "" # AZBLOB_OBJECT_DISK_PATH, path for backup of part from `azure_blob_storage` object disk, if disk present, then shall not be zero and shall not be prefixed by `path` compression_level: 1 # AZBLOB_COMPRESSION_LEVEL compression_format: tar # AZBLOB_COMPRESSION_FORMAT, allowed values tar, lz4, bzip2, gzip, sz, xz, brortli, zstd, `none` for upload data part folders as is @@ -437,26 +204,30 @@ azblob: buffer_size: 0 # AZBLOB_BUFFER_SIZE, if less or eq 0 then it is calculated as max_file_size / max_parts_count, between 2Mb and 4Mb max_parts_count: 10000 # AZBLOB_MAX_PARTS_COUNT, number of parts for AZBLOB uploads, for properly calculate buffer size max_buffers: 3 # AZBLOB_MAX_BUFFERS + debug: false # AZBLOB_DEBUG s3: access_key: "" # S3_ACCESS_KEY secret_key: "" # S3_SECRET_KEY bucket: "" # S3_BUCKET endpoint: "" # S3_ENDPOINT region: us-east-1 # S3_REGION - acl: private # S3_ACL + # AWS changed S3 defaults in April 2023 so that all new buckets have ACL disabled: https://aws.amazon.com/blogs/aws/heads-up-amazon-s3-security-changes-are-coming-in-april-of-2023/ + # They also recommend that ACLs are disabled: https://docs.aws.amazon.com/AmazonS3/latest/userguide/ensure-object-ownership.html + # use `acl: ""` if you see "api error AccessControlListNotSupported: The bucket does not allow ACLs" + acl: private # S3_ACL assume_role_arn: "" # S3_ASSUME_ROLE_ARN force_path_style: false # S3_FORCE_PATH_STYLE - path: "" # S3_PATH, `system.macros` values could be applied as {macro_name} + path: "" # S3_PATH, `system.macros` values can be applied as {macro_name} object_disk_path: "" # S3_OBJECT_DISK_PATH, path for backup of part from `s3` object disk, if disk present, then shall not be zero and shall not be prefixed by `path` disable_ssl: false # S3_DISABLE_SSL compression_level: 1 # S3_COMPRESSION_LEVEL compression_format: tar # S3_COMPRESSION_FORMAT, allowed values tar, lz4, bzip2, gzip, sz, xz, brortli, zstd, `none` for upload data part folders as is - # look details in https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html + # look at details in https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html sse: "" # S3_SSE, empty (default), AES256, or aws:kms - sse_kms_key_id: "" # S3_SSE_KMS_KEY_ID, if S3_SSE is aws:kms then specifies the ID of the Amazon Web Services Key Management Service sse_customer_algorithm: "" # S3_SSE_CUSTOMER_ALGORITHM, encryption algorithm, for example, AES256 - sse_customer_key: "" # S3_SSE_CUSTOMER_KEY, customer-provided encryption key - sse_customer_key_md5: "" # S3_SSE_CUSTOMER_KEY_MD5, 128-bit MD5 digest of the encryption key according to RFC 1321 + sse_customer_key: "" # S3_SSE_CUSTOMER_KEY, customer-provided encryption key use `openssl rand 32 > aws_sse.key` and `cat aws_sse.key | base64` + sse_customer_key_md5: "" # S3_SSE_CUSTOMER_KEY_MD5, 128-bit MD5 digest of the encryption key according to RFC 1321 use `cat aws_sse.key | openssl dgst -md5 -binary | base64` + sse_kms_key_id: "" # S3_SSE_KMS_KEY_ID, if S3_SSE is aws:kms then specifies the ID of the Amazon Web Services Key Management Service sse_kms_encryption_context: "" # S3_SSE_KMS_ENCRYPTION_CONTEXT, base64-encoded UTF-8 string holding a JSON with the encryption context # Specifies the Amazon Web Services KMS Encryption Context to use for object encryption. # This is a collection of non-secret key-value pairs that represent additional authenticated data. @@ -469,36 +240,46 @@ s3: part_size: 0 # S3_PART_SIZE, if less or eq 0 then it is calculated as max_file_size / max_parts_count, between 5MB and 5Gb max_parts_count: 10000 # S3_MAX_PARTS_COUNT, number of parts for S3 multipart uploads allow_multipart_download: false # S3_ALLOW_MULTIPART_DOWNLOAD, allow faster download and upload speeds, but will require additional disk space, download_concurrency * part size in worst case + checksum_algorithm: "" # S3_CHECKSUM_ALGORITHM, use it when you use object lock which allow to avoid delete keys from bucket until some timeout after creation, use CRC32 as fastest # S3_OBJECT_LABELS, allow setup metadata for each object during upload, use {macro_name} from system.macros and {backupName} for current backup name # The format for this env variable is "key1:value1,key2:value2". For YAML please continue using map syntax object_labels: {} # S3_CUSTOM_STORAGE_CLASS_MAP, allow setup storage class depending on the backup name regexp pattern, format nameRegexp > className custom_storage_class_map: {} + # S3_REQUEST_PAYER, define who will pay to request, look https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html for details, possible values requester, if empty then bucket owner + request_payer: "" debug: false # S3_DEBUG gcs: credentials_file: "" # GCS_CREDENTIALS_FILE credentials_json: "" # GCS_CREDENTIALS_JSON credentials_json_encoded: "" # GCS_CREDENTIALS_JSON_ENCODED + # look https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create how to get HMAC keys for access to bucket + embedded_access_key: "" # GCS_EMBEDDED_ACCESS_KEY, use it when `use_embedded_backup_restore: true`, `embedded_backup_disk: ""`, `remote_storage: gcs` + embedded_secret_key: "" # GCS_EMBEDDED_SECRET_KEY, use it when `use_embedded_backup_restore: true`, `embedded_backup_disk: ""`, `remote_storage: gcs` + skip_credentials: false # GCS_SKIP_CREDENTIALS, skip add credentials to requests to allow anonymous access to bucket + endpoint: "" # GCS_ENDPOINT, use it for custom GCS endpoint/compatible storage. For example, when using custom endpoint via private service connect bucket: "" # GCS_BUCKET - path: "" # GCS_PATH, `system.macros` values could be applied as {macro_name} + path: "" # GCS_PATH, `system.macros` values can be applied as {macro_name} object_disk_path: "" # GCS_OBJECT_DISK_PATH, path for backup of part from `s3` object disk (clickhouse support only gcs over s3 protocol), if disk present, then shall not be zero and shall not be prefixed by `path` compression_level: 1 # GCS_COMPRESSION_LEVEL compression_format: tar # GCS_COMPRESSION_FORMAT, allowed values tar, lz4, bzip2, gzip, sz, xz, brortli, zstd, `none` for upload data part folders as is storage_class: STANDARD # GCS_STORAGE_CLASS - + chunk_size: 0 # GCS_CHUNK_SIZE, default 16 * 1024 * 1024 (16MB) + client_pool_size: 500 # GCS_CLIENT_POOL_SIZE, default max(upload_concurrency, download concurrency) * 3, should be at least 3 times bigger than `UPLOAD_CONCURRENCY` or `DOWNLOAD_CONCURRENCY` in each upload and download case to avoid stuck # GCS_OBJECT_LABELS, allow setup metadata for each object during upload, use {macro_name} from system.macros and {backupName} for current backup name # The format for this env variable is "key1:value1,key2:value2". For YAML please continue using map syntax object_labels: {} # GCS_CUSTOM_STORAGE_CLASS_MAP, allow setup storage class depends on backup name regexp pattern, format nameRegexp > className custom_storage_class_map: {} debug: false # GCS_DEBUG + force_http: false # GCS_FORCE_HTTP cos: url: "" # COS_URL timeout: 2m # COS_TIMEOUT secret_id: "" # COS_SECRET_ID secret_key: "" # COS_SECRET_KEY - path: "" # COS_PATH, `system.macros` values could be applied as {macro_name} + path: "" # COS_PATH, `system.macros` values can be applied as {macro_name} compression_format: tar # COS_COMPRESSION_FORMAT, allowed values tar, lz4, bzip2, gzip, sz, xz, brortli, zstd, `none` for upload data part folders as is compression_level: 1 # COS_COMPRESSION_LEVEL ftp: @@ -507,7 +288,8 @@ ftp: username: "" # FTP_USERNAME password: "" # FTP_PASSWORD tls: false # FTP_TLS - path: "" # FTP_PATH, `system.macros` values could be applied as {macro_name} + tls_skip_verify: false # FTP_TLS_SKIP_VERIFY + path: "" # FTP_PATH, `system.macros` values can be applied as {macro_name} compression_format: tar # FTP_COMPRESSION_FORMAT, allowed values tar, lz4, bzip2, gzip, sz, xz, brortli, zstd, `none` for upload data part folders as is compression_level: 1 # FTP_COMPRESSION_LEVEL debug: false # FTP_DEBUG @@ -517,7 +299,7 @@ sftp: password: "" # SFTP_PASSWORD port: 22 # SFTP_PORT key: "" # SFTP_KEY - path: "" # SFTP_PATH, `system.macros` values could be applied as {macro_name} + path: "" # SFTP_PATH, `system.macros` values can be applied as {macro_name} concurrency: 1 # SFTP_CONCURRENCY compression_format: tar # SFTP_COMPRESSION_FORMAT, allowed values tar, lz4, bzip2, gzip, sz, xz, brortli, zstd, `none` for upload data part folders as is compression_level: 1 # SFTP_COMPRESSION_LEVEL @@ -546,66 +328,70 @@ api: allow_parallel: false # API_ALLOW_PARALLEL, enable parallel operations, this allows for significant memory allocation and spawns go-routines, don't enable it if you are not sure create_integration_tables: false # API_CREATE_INTEGRATION_TABLES, create `system.backup_list` and `system.backup_actions` complete_resumable_after_restart: true # API_COMPLETE_RESUMABLE_AFTER_RESTART, after API server startup, if `/var/lib/clickhouse/backup/*/(upload|download).state` present, then operation will continue in the background + watch_is_main_process: false # WATCH_IS_MAIN_PROCESS, treats 'watch' command as a main api process, if it is stopped unexpectedly, api server is also stopped. Does not stop api server if 'watch' command canceled by the user. ``` ## Concurrency, CPU and Memory usage recommendation -`upload_concurrency` and `download concurrency` define how much parallel download / upload go-routines will start independently of the remote storage type. -In 1.3.0+ it means how many parallel data parts will be uploaded, assuming `upload_by_part` and `download_by_part` are `true` (which is default value). +`upload_concurrency` and `download_concurrency` define how many parallel download / upload go-routines will start independently of the remote storage type. +In 1.3.0+ it means how many parallel data parts will be uploaded, assuming `upload_by_part` and `download_by_part` are `true` (which is the default value). -`concurrency` in `s3` section means how much concurrent `upload` streams will run during multipart upload in each upload go-routine -High value for `S3_CONCURRENCY` and high value for `S3_PART_SIZE` will allocate a lot of memory for buffers inside AWS golang SDK. +`concurrency` in the `s3` section means how many concurrent `upload` streams will run during multipart upload in each upload go-routine. +A high value for `S3_CONCURRENCY` and a high value for `S3_PART_SIZE` will allocate a lot of memory for buffers inside the AWS golang SDK. -`concurrency` in `sftp` section means how many concurrent request will be used for `upload` and `download` for each file. +`concurrency` in the `sftp` section means how many concurrent request will be used for `upload` and `download` for each file. -`compression_format`, a good default is `tar` for using less CPU. In most cases the data in clickhouse is already compressed, so you may not get a lot of space savings when double compress. +For `compression_format`, a good default is `tar`, which uses less CPU. In most cases the data in clickhouse is already compressed, so you may not get a lot of space savings when compressing already-compressed data. ## remote_storage: custom -All custom commands could use go-template language, for example, you can use `{{ .cfg.* }}` `{{ .backupName }}` `{{ .diffFromRemote }}`. -Custom `list_command` shall return JSON which is compatible with `metadata.Backup` type with [JSONEachRow](https://clickhouse.com/docs/en/interfaces/formats/#jsoneachrow) format. -For examples, see [restic](https://github.com/Altinity/clickhouse-backup/tree/master/test/integration/restic/), [rsync](https://github.com/Altinity/clickhouse-backup/tree/master/test/integration/rsync/) and [kopia](https://github.com/Altinity/clickhouse-backup/tree/master/test/integration/kopia/). Feel free to add yours too. +All custom commands use the go-template language. For example, you can use `{{ .cfg.* }}` `{{ .backupName }}` `{{ .diffFromRemote }}`. +A custom `list_command` returns JSON which is compatible with the `metadata.BackupMetadata` type with [JSONEachRow](https://clickhouse.com/docs/en/interfaces/formats/#jsoneachrow) format. +For examples, see [restic](https://github.com/Altinity/clickhouse-backup/tree/master/test/integration/restic/), [rsync](https://github.com/Altinity/clickhouse-backup/tree/master/test/integration/rsync/) and [kopia](https://github.com/Altinity/clickhouse-backup/tree/master/test/integration/kopia/). Feel free to add yours custom storage. ## ATTENTION! -Never change files permissions in `/var/lib/clickhouse/backup`. +**Never change file permissions in `/var/lib/clickhouse/backup`.** This path contains hard links. Permissions on all hard links to the same data on disk are always identical. That means that if you change the permissions/owner/attributes on a hard link in backup path, permissions on files with which ClickHouse works will be changed too. -That might lead to data corruption. +That can lead to data corruption. ## API Use the `clickhouse-backup server` command to run as a REST API server. In general, the API attempts to mirror the CLI commands. -> **GET /** +### GET / List all current applicable HTTP routes -> **POST /** -> **POST /restart** +### POST / + +### POST /restart Restart HTTP server, close all current connections, close listen socket, open listen socket again, all background go-routines breaks with contexts -> **GET /backup/kill** +### GET /backup/kill Kill selected command from `GET /backup/actions` command list, kill process should be near immediate, but some go-routines (upload one data part) could continue to run. -- Optional query argument `command` may contain the command name to kill, or if it is omitted then kill the last "in progress" command. +- Optional query argument `command` may contain the command name to kill, or if it is omitted then kill the first "in progress" command. -> **GET /backup/tables** +### GET /backup/tables Print list of tables: `curl -s localhost:7171/backup/tables | jq .`, exclude pattern matched tables from `skip_tables` configuration parameters -- Optional query argument `table` works the same as the `--table value` CLI argument. +- Optional query argument `table` works the same as the `--table=pattern` CLI argument. +- Optional query argument `remote_backup`works the same as `--remote-backup=name` CLI argument. -> **GET /backup/tables/all** +### GET /backup/tables/all Print list of tables: `curl -s localhost:7171/backup/tables/all | jq .`, ignore `skip_tables` configuration parameters. - Optional query argument `table` works the same as the `--table value` CLI argument. +- Optional query argument `remote_backup`works the same as `--remote-backup=name` CLI argument. -> **POST /backup/create** +### POST /backup/create Create new backup: `curl -s localhost:7171/backup/create -X POST | jq .` @@ -618,9 +404,9 @@ Create new backup: `curl -s localhost:7171/backup/create -X POST | jq .` - Optional query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success","error":"not empty when error happens"}`. - Additional example: `curl -s 'localhost:7171/backup/create?table=default.billing&name=billing_test' -X POST` -Note: this operation is async, so the API will return once the operation has started. +Note: this operation is asynchronous, so the API will return once the operation has started. -> **POST /backup/watch** +### POST /backup/watch Run background watch process and create full+incremental backups sequence: `curl -s localhost:7171/backup/watch -X POST | jq .` You can't run watch twice with the same parameters even when `allow_parallel: true` @@ -635,21 +421,22 @@ You can't run watch twice with the same parameters even when `allow_parallel: tr - Optional query argument `configs` works the same as the `--configs` CLI argument (backup configs). - Additional example: `curl -s 'localhost:7171/backup/watch?table=default.billing&watch_interval=1h&full_interval=24h' -X POST` -Note: this operation is async and can stop only with `kill -s SIGHUP $(pgrep -f clickhouse-backup)` or call `/restart`, `/backup/kill`. The API will return immediately once the operation has started. +Note: this operation is asynchronous and can only be stopped with `kill -s SIGHUP $(pgrep -f clickhouse-backup)` or call `/restart`, `/backup/kill`. The API will return immediately once the operation has started. -> **POST /backup/clean** +### POST /backup/clean Clean the `shadow` folders using all available paths from `system.disks` -> **POST /backup/clean/remote_broken** +### POST /backup/clean/remote_broken Remove Note: this operation is sync, and could take a lot of time, increase http timeouts during call -> **POST /backup/upload** +### POST /backup/upload Upload backup to remote storage: `curl -s localhost:7171/backup/upload/ -X POST | jq .` +- Optional query argument `delete-source` works the same as the `--delete-source` CLI argument. - Optional query argument `diff-from` works the same as the `--diff-from` CLI argument. - Optional query argument `diff-from-remote` works the same as the `--diff-from-remote` CLI argument. - Optional query argument `table` works the same as the `--table value` CLI argument. @@ -658,18 +445,18 @@ Upload backup to remote storage: `curl -s localhost:7171/backup/upload/ **GET /backup/list/{where}** +### GET /backup/list/{where} -Print list of backups: `curl -s localhost:7171/backup/list | jq .` -Print list only local backups: `curl -s localhost:7171/backup/list/local | jq .` -Print list only remote backups: `curl -s localhost:7171/backup/list/remote | jq .` +Print a list of backups: `curl -s localhost:7171/backup/list | jq .` +Print a list of only local backups: `curl -s localhost:7171/backup/list/local | jq .` +Print a list of only remote backups: `curl -s localhost:7171/backup/list/remote | jq .` Note: The `Size` field will not be set for the local backups that have just been created or are in progress. Note: The `Size` field will not be set for the remote backups with upload status in progress. -> **POST /backup/download** +### POST /backup/download Download backup from remote storage: `curl -s localhost:7171/backup/download/ -X POST | jq .` @@ -679,9 +466,9 @@ Download backup from remote storage: `curl -s localhost:7171/backup/download/ **POST /backup/restore** +### POST /backup/restore Create schema and restore data from backup: `curl -s localhost:7171/backup/restore/ -X POST | jq .` @@ -694,23 +481,24 @@ Create schema and restore data from backup: `curl -s localhost:7171/backup/resto - Optional query argument `rbac` works the same as the `--rbac` CLI argument (restore RBAC). - Optional query argument `configs` works the same as the `--configs` CLI argument (restore configs). - Optional query argument `restore_database_mapping` works the same as the `--restore-database-mapping` CLI argument. +- Optional query argument `restore_table_mapping` works the same as the `--restore-table-mapping` CLI argument. - Optional query argument `callback` allow pass callback URL which will call with POST with `application/json` with payload `{"status":"error|success","error":"not empty when error happens"}`. -> **POST /backup/delete** +### POST /backup/delete Delete specific remote backup: `curl -s localhost:7171/backup/delete/remote/ -X POST | jq .` Delete specific local backup: `curl -s localhost:7171/backup/delete/local/ -X POST | jq .` -> **GET /backup/status** +### GET /backup/status -Display list of currently running async operation: `curl -s localhost:7171/backup/status | jq .` +Display list of currently running asynchronous operations: `curl -s localhost:7171/backup/status | jq .` -> **POST /backup/actions** +### POST /backup/actions Execute multiple backup actions: `curl -X POST -d '{"command":"create test_backup"}' -s localhost:7171/backup/actions` -> **GET /backup/actions** +### GET /backup/actions Display a list of all operations from start of API server: `curl -s localhost:7171/backup/actions | jq .` @@ -721,7 +509,7 @@ Display a list of all operations from start of API server: `curl -s localhost:71 ### S3 -In order to make backups to S3, the following permissions shall be set: +In order to make backups to S3, the following permissions should be set: ```json { @@ -732,14 +520,18 @@ In order to make backups to S3, the following permissions shall be set: "Effect": "Allow", "Action": [ "s3:PutObject", - "s3:GetObject" + "s3:GetObject", + "s3:DeleteObject" ], "Resource": "arn:aws:s3:::BUCKET_NAME/*" }, { "Sid": "clickhouse-backup-s3-access-to-bucket", "Effect": "Allow", - "Action": "s3:ListBucket", + "Action": [ + "s3:ListBucket", + "s3:GetBucketVersioning" + ], "Resource": "arn:aws:s3:::BUCKET_NAME" } ] @@ -768,6 +560,326 @@ if [[ $exit_code != 0 ]]; then fi ``` +## Common CLI Usage + +### CLI command - tables +``` +NAME: + clickhouse-backup tables - List of tables, exclude skip_tables + +USAGE: + clickhouse-backup tables [--tables=.
] [--remote-backup=] [--all] + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --all, -a Print table even when match with skip_tables pattern + --table value, --tables value, -t value List tables only match with table name patterns, separated by comma, allow ? and * as wildcard + --remote-backup value List tables from remote backup + +``` +### CLI command - create +``` +NAME: + clickhouse-backup create - Create new backup + +USAGE: + clickhouse-backup create [-t, --tables=.
] [--partitions=] [-s, --schema] [--rbac] [--configs] [--skip-check-parts-columns] + +DESCRIPTION: + Create new backup + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --table value, --tables value, -t value Create backup only matched with table name patterns, separated by comma, allow ? and * as wildcard + --diff-from-remote value Create incremental embedded backup or upload incremental object disk data based on other remote backup name + --partitions partition_id Create backup only for selected partition names, separated by comma +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ + --schema, -s Backup schemas only, will skip data + --rbac, --backup-rbac, --do-backup-rbac Backup RBAC related objects + --configs, --backup-configs, --do-backup-configs Backup 'clickhouse-server' configuration files + --rbac-only Backup RBAC related objects only, will skip backup data, will backup schema only if --schema added + --configs-only Backup 'clickhouse-server' configuration files only, will skip backup data, will backup schema only if --schema added + --skip-check-parts-columns Skip check system.parts_columns to disallow backup inconsistent column types for data parts + +``` +### CLI command - create_remote +``` +NAME: + clickhouse-backup create_remote - Create and upload new backup + +USAGE: + clickhouse-backup create_remote [-t, --tables=.
] [--partitions=] [--diff-from=] [--diff-from-remote=] [--schema] [--rbac] [--configs] [--resumable] [--skip-check-parts-columns] + +DESCRIPTION: + Create and upload + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --table value, --tables value, -t value Create and upload backup only matched with table name patterns, separated by comma, allow ? and * as wildcard + --partitions partition_id Create and upload backup only for selected partition names, separated by comma +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ + --diff-from value Local backup name which used to upload current backup as incremental + --diff-from-remote value Remote backup name which used to upload current backup as incremental + --schema, -s Backup and upload metadata schema only, will skip data backup + --rbac, --backup-rbac, --do-backup-rbac Backup and upload RBAC related objects + --configs, --backup-configs, --do-backup-configs Backup and upload 'clickhouse-server' configuration files + --rbac-only Backup RBAC related objects only, will skip backup data, will backup schema only if --schema added + --configs-only Backup 'clickhouse-server' configuration files only, will skip backup data, will backup schema only if --schema added + --resume, --resumable Save intermediate upload state and resume upload if backup exists on remote storage, ignore when 'remote_storage: custom' or 'use_embedded_backup_restore: true' + --skip-check-parts-columns Skip check system.parts_columns to disallow backup inconsistent column types for data parts + --delete, --delete-source, --delete-local explicitly delete local backup during upload + +``` +### CLI command - upload +``` +NAME: + clickhouse-backup upload - Upload backup to remote storage + +USAGE: + clickhouse-backup upload [-t, --tables=.
] [--partitions=] [-s, --schema] [--diff-from=] [--diff-from-remote=] [--resumable] + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --diff-from value Local backup name which used to upload current backup as incremental + --diff-from-remote value Remote backup name which used to upload current backup as incremental + --table value, --tables value, -t value Upload data only for matched table name patterns, separated by comma, allow ? and * as wildcard + --partitions partition_id Upload backup only for selected partition names, separated by comma +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ + --schema, -s Upload schemas only + --resume, --resumable Save intermediate upload state and resume upload if backup exists on remote storage, ignored with 'remote_storage: custom' or 'use_embedded_backup_restore: true' + --delete, --delete-source, --delete-local explicitly delete local backup during upload + +``` +### CLI command - list +``` +NAME: + clickhouse-backup list - List of backups + +USAGE: + clickhouse-backup list [all|local|remote] [latest|previous] + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + +``` +### CLI command - download +``` +NAME: + clickhouse-backup download - Download backup from remote storage + +USAGE: + clickhouse-backup download [-t, --tables=.
] [--partitions=] [-s, --schema] [--resumable] + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --table value, --tables value, -t value Download objects which matched with table name patterns, separated by comma, allow ? and * as wildcard + --partitions partition_id Download backup data only for selected partition names, separated by comma +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ + --schema, -s Download schema only + --resume, --resumable Save intermediate download state and resume download if backup exists on local storage, ignored with 'remote_storage: custom' or 'use_embedded_backup_restore: true' + +``` +### CLI command - restore +``` +NAME: + clickhouse-backup restore - Create schema and restore data from backup + +USAGE: + clickhouse-backup restore [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--tm, --restore-table-mapping=:[,<...>]] [--partitions=] [-s, --schema] [-d, --data] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --table value, --tables value, -t value Restore only database and objects which matched with table name patterns, separated by comma, allow ? and * as wildcard + --restore-database-mapping value, -m value Define the rule to restore data. For the database not defined in this struct, the program will not deal with it. + --restore-table-mapping value, --tm value Define the rule to restore data. For the table not defined in this struct, the program will not deal with it. + --partitions partition_id Restore backup only for selected partition names, separated by comma +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ + --schema, -s Restore schema only + --data, -d Restore data only + --rm, --drop Drop exists schema objects before restore + -i, --ignore-dependencies Ignore dependencies when drop exists schema objects + --rbac, --restore-rbac, --do-restore-rbac Restore RBAC related objects + --configs, --restore-configs, --do-restore-configs Restore 'clickhouse-server' CONFIG related files + --rbac-only Restore RBAC related objects only, will skip backup data, will backup schema only if --schema added + --configs-only Restore 'clickhouse-server' configuration files only, will skip backup data, will backup schema only if --schema added + +``` +### CLI command - restore_remote +``` +NAME: + clickhouse-backup restore_remote - Download and restore + +USAGE: + clickhouse-backup restore_remote [--schema] [--data] [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--tm, --restore-table-mapping=:[,<...>]] [--partitions=] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] [--skip-rbac] [--skip-configs] [--resumable] + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --table value, --tables value, -t value Download and restore objects which matched with table name patterns, separated by comma, allow ? and * as wildcard + --restore-database-mapping value, -m value Define the rule to restore data. For the database not defined in this struct, the program will not deal with it. + --restore-table-mapping value, --tm value Define the rule to restore data. For the table not defined in this struct, the program will not deal with it. + --partitions partition_id Download and restore backup only for selected partition names, separated by comma +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ + --schema, -s Download and Restore schema only + --data, -d Download and Restore data only + --rm, --drop Drop schema objects before restore + -i, --ignore-dependencies Ignore dependencies when drop exists schema objects + --rbac, --restore-rbac, --do-restore-rbac Download and Restore RBAC related objects + --configs, --restore-configs, --do-restore-configs Download and Restore 'clickhouse-server' CONFIG related files + --rbac-only Restore RBAC related objects only, will skip backup data, will backup schema only if --schema added + --configs-only Restore 'clickhouse-server' configuration files only, will skip backup data, will backup schema only if --schema added + --resume, --resumable Save intermediate upload state and resume upload if backup exists on remote storage, ignored with 'remote_storage: custom' or 'use_embedded_backup_restore: true' + +``` +### CLI command - delete +``` +NAME: + clickhouse-backup delete - Delete specific backup + +USAGE: + clickhouse-backup delete + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + +``` +### CLI command - default-config +``` +NAME: + clickhouse-backup default-config - Print default config + +USAGE: + clickhouse-backup default-config [command options] [arguments...] + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + +``` +### CLI command - print-config +``` +NAME: + clickhouse-backup print-config - Print current config merged with environment variables + +USAGE: + clickhouse-backup print-config [command options] [arguments...] + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + +``` +### CLI command - clean +``` +NAME: + clickhouse-backup clean - Remove data in 'shadow' folder from all 'path' folders available from 'system.disks' + +USAGE: + clickhouse-backup clean [command options] [arguments...] + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + +``` +### CLI command - clean_remote_broken +``` +NAME: + clickhouse-backup clean_remote_broken - Remove all broken remote backups + +USAGE: + clickhouse-backup clean_remote_broken [command options] [arguments...] + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + +``` +### CLI command - watch +``` +NAME: + clickhouse-backup watch - Run infinite loop which create full + incremental backup sequence to allow efficient backup sequences + +USAGE: + clickhouse-backup watch [--watch-interval=1h] [--full-interval=24h] [--watch-backup-name-template=shard{shard}-{type}-{time:20060102150405}] [-t, --tables=.
] [--partitions=] [--schema] [--rbac] [--configs] [--skip-check-parts-columns] + +DESCRIPTION: + Execute create_remote + delete local, create full backup every `--full-interval`, create and upload incremental backup every `--watch-interval` use previous backup as base with `--diff-from-remote` option, use `backups_to_keep_remote` config option for properly deletion remote backups, will delete old backups which not have references from other backups + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --watch-interval value Interval for run 'create_remote' + 'delete local' for incremental backup, look format https://pkg.go.dev/time#ParseDuration + --full-interval value Interval for run 'create_remote'+'delete local' when stop create incremental backup sequence and create full backup, look format https://pkg.go.dev/time#ParseDuration + --watch-backup-name-template value Template for new backup name, could contain names from system.macros, {type} - full or incremental and {time:LAYOUT}, look to https://go.dev/src/time/format.go for layout examples + --table value, --tables value, -t value Create and upload only objects which matched with table name patterns, separated by comma, allow ? and * as wildcard + --partitions partition_id Partitions names, separated by comma +If PARTITION BY clause returns numeric not hashed values for partition_id field in system.parts table, then use --partitions=partition_id1,partition_id2 format +If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format +If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format +If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:* +Values depends on field types in your table, use single quotes for String and Date/DateTime related types +Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/ + --schema, -s Schemas only + --rbac, --backup-rbac, --do-backup-rbac Backup RBAC related objects only + --configs, --backup-configs, --do-backup-configs Backup `clickhouse-server' configuration files only + --skip-check-parts-columns Skip check system.parts_columns to disallow backup inconsistent column types for data parts + +``` +### CLI command - server +``` +NAME: + clickhouse-backup server - Run API server + +USAGE: + clickhouse-backup server [command options] [arguments...] + +OPTIONS: + --config value, -c value Config 'FILE' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG] + --environment-override value, --env value override any environment variable via CLI parameter + --watch Run watch go-routine for 'create_remote' + 'delete local', after API server startup + --watch-interval value Interval for run 'create_remote' + 'delete local' for incremental backup, look format https://pkg.go.dev/time#ParseDuration + --full-interval value Interval for run 'create_remote'+'delete local' when stop create incremental backup sequence and create full backup, look format https://pkg.go.dev/time#ParseDuration + --watch-backup-name-template value Template for new backup name, could contain names from system.macros, {type} - full or incremental and {time:LAYOUT}, look to https://go.dev/src/time/format.go for layout examples + ``` ### More use cases of clickhouse-backup @@ -775,14 +887,13 @@ fi - [How to convert MergeTree to ReplicatedMergeTree](Examples.md#how-to-convert-mergetree-to-replicatedmergetree) - [How to store backups on NFS or another server](Examples.md#how-to-store-backups-on-nfs-backup-drive-or-another-server-via-sftp) - [How to move data to another clickhouse server](Examples.md#how-to-move-data-to-another-clickhouse-server) -- [How to reduce number of partitions](Examples.md#How-to-reduce-number-of-partitions) -- [How to monitor that backups created and uploaded correctly](Examples.md#how-to-monitor-that-backups-created-and-uploaded-correctly) -- [How to make backup / restore sharded cluster](Examples.md#how-to-make-backup--restore-sharded-cluster) -- [How to make backup sharded cluster with Ansible](Examples.md#how-to-make-backup-sharded-cluster-with-ansible) -- [How to make back up database with several terabytes of data](Examples.md#how-to-make-backup-database-with-several-terabytes-of-data) +- [How to monitor that backups created and uploaded correctly](Examples.md#how-to-monitor-that-backups-were-created-and-uploaded-correctly) +- [How to back up / restore a sharded cluster](Examples.md#how-to-back-up--restore-a-sharded-cluster) +- [How to back up a sharded cluster with Ansible](Examples.md#how-to-back-up-a-sharded-cluster-with-ansible) +- [How to back up a database with several terabytes of data](Examples.md#how-to-back-up-a-database-with-several-terabytes-of-data) - [How to use clickhouse-backup in Kubernetes](Examples.md#how-to-use-clickhouse-backup-in-kubernetes) -- [How do incremental backups work to remote storage](Examples.md#how-do-incremental-backups-work-to-remote-storage) -- [How to watch backups work](Examples.md#how-to-work-watch-command) +- [How to do incremental backups work to remote storage](Examples.md#how-incremental-backups-work-with-remote-storage) +- [How to watch backups work](Examples.md#how-to-watch-backups-work) ## Original Author Altinity wants to thank @[AlexAkulov](https://github.com/AlexAkulov) for creating this tool and for his valuable contributions. diff --git a/Vagrantfile b/Vagrantfile index 12eae47a..6e40276d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -101,7 +101,7 @@ Vagrant.configure(2) do |config| apt-get install --no-install-recommends -y clickhouse-client clickhouse-server # golang - export GOLANG_VERSION=1.21 + export GOLANG_VERSION=1.22 apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 52B59B1571A79DBC054901C0F6BC817356A3D45E add-apt-repository ppa:longsleep/golang-backports apt-get install --no-install-recommends -y golang-${GOLANG_VERSION} diff --git a/cmd/clickhouse-backup/main.go b/cmd/clickhouse-backup/main.go index a883ea7a..07e110db 100644 --- a/cmd/clickhouse-backup/main.go +++ b/cmd/clickhouse-backup/main.go @@ -7,15 +7,15 @@ import ( "os" "strings" - "github.com/Altinity/clickhouse-backup/pkg/backup" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/server" - "github.com/Altinity/clickhouse-backup/pkg/status" - "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/rs/zerolog/pkgerrors" "github.com/urfave/cli" + + "github.com/Altinity/clickhouse-backup/v2/pkg/backup" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/server" + "github.com/Altinity/clickhouse-backup/v2/pkg/status" ) var ( @@ -27,7 +27,7 @@ var ( func main() { zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack - consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout, NoColor: true, TimeFormat: "2006-01-02 15:04:05.000"} + consoleWriter := zerolog.ConsoleWriter{Out: os.Stderr, NoColor: true, TimeFormat: "2006-01-02 15:04:05.000"} //diodeWriter := diode.NewWriter(consoleWriter, 4096, 10*time.Millisecond, func(missed int) { // fmt.Printf("Logger Dropped %d messages", missed) //}) @@ -47,10 +47,16 @@ func main() { } cliapp.Flags = []cli.Flag{ cli.StringFlag{ - Name: "config, c", - Value: config.DefaultConfigPath, - Usage: "Config 'FILE' name.", - EnvVar: "CLICKHOUSE_BACKUP_CONFIG", + Name: "config, c", + Value: config.DefaultConfigPath, + Usage: "Config 'FILE' name.", + EnvVar: "CLICKHOUSE_BACKUP_CONFIG", + Required: false, + }, + cli.StringSliceFlag{ + Name: "environment-override, env", + Usage: "override any environment variable via CLI parameter", + Required: false, }, cli.IntFlag{ Name: "command-id", @@ -75,21 +81,26 @@ func main() { { Name: "tables", Usage: "List of tables, exclude skip_tables", - UsageText: "clickhouse-backup tables [-t, --tables=.
]] [--all]", + UsageText: "clickhouse-backup tables [--tables=.
] [--remote-backup=] [--all]", Action: func(c *cli.Context) error { b := backup.NewBackuper(config.GetConfigFromCli(c)) - return b.PrintTables(c.Bool("all"), c.String("table")) + return b.PrintTables(c.Bool("all"), c.String("table"), c.String("remote-backup")) }, Flags: append(cliapp.Flags, cli.BoolFlag{ Name: "all, a", Hidden: false, - Usage: "print table even when match with skip_tables pattern", + Usage: "Print table even when match with skip_tables pattern", }, cli.StringFlag{ Name: "table, tables, t", Hidden: false, - Usage: "list tables only match with table name patterns, separated by comma, allow ? and * as wildcard", + Usage: "List tables only match with table name patterns, separated by comma, allow ? and * as wildcard", + }, + cli.StringFlag{ + Name: "remote-backup", + Hidden: false, + Usage: "List tables from remote backup", }, ), }, @@ -100,23 +111,29 @@ func main() { Description: "Create new backup", Action: func(c *cli.Context) error { b := backup.NewBackuper(config.GetConfigFromCli(c)) - return b.CreateBackup(c.Args().First(), c.String("t"), c.StringSlice("partitions"), c.Bool("s"), c.Bool("rbac"), c.Bool("rbac-only"), c.Bool("configs"), c.Bool("configs-only"), c.Bool("skip-check-parts-columns"), version, c.Int("command-id")) + return b.CreateBackup(c.Args().First(), c.String("diff-from-remote"), c.String("t"), c.StringSlice("partitions"), c.Bool("s"), c.Bool("rbac"), c.Bool("rbac-only"), c.Bool("configs"), c.Bool("configs-only"), c.Bool("skip-check-parts-columns"), version, c.Int("command-id")) }, Flags: append(cliapp.Flags, cli.StringFlag{ Name: "table, tables, t", Hidden: false, - Usage: "create backup only matched with table name patterns, separated by comma, allow ? and * as wildcard", + Usage: "Create backup only matched with table name patterns, separated by comma, allow ? and * as wildcard", + }, + cli.StringFlag{ + Name: "diff-from-remote", + Hidden: false, + Usage: "Create incremental embedded backup or upload incremental object disk data based on other remote backup name", }, cli.StringSliceFlag{ Name: "partitions", Hidden: false, - Usage: "create backup only for selected partition names, separated by comma\n" + - "if PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + - "if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + - "if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + - "values depends on field types in your table, use single quote for String and Date/DateTime related types\n" + - "look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", + Usage: "Create backup only for selected partition names, separated by comma\n" + + "If PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + + "If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + + "If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + + "If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:*\n" + + "Values depends on field types in your table, use single quotes for String and Date/DateTime related types\n" + + "Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", }, cli.BoolFlag{ Name: "schema, s", @@ -146,7 +163,7 @@ func main() { cli.BoolFlag{ Name: "skip-check-parts-columns", Hidden: false, - Usage: "skip check system.parts_columns to disallow backup inconsistent column types for data parts", + Usage: "Skip check system.parts_columns to disallow backup inconsistent column types for data parts", }, ), }, @@ -157,33 +174,34 @@ func main() { Description: "Create and upload", Action: func(c *cli.Context) error { b := backup.NewBackuper(config.GetConfigFromCli(c)) - return b.CreateToRemote(c.Args().First(), c.String("diff-from"), c.String("diff-from-remote"), c.String("t"), c.StringSlice("partitions"), c.Bool("s"), c.Bool("rbac"), c.Bool("rbac-only"), c.Bool("configs"), c.Bool("configs-only"), c.Bool("resume"), c.Bool("skip-check-parts-columns"), version, c.Int("command-id")) + return b.CreateToRemote(c.Args().First(), c.Bool("delete-source"), c.String("diff-from"), c.String("diff-from-remote"), c.String("t"), c.StringSlice("partitions"), c.Bool("s"), c.Bool("rbac"), c.Bool("rbac-only"), c.Bool("configs"), c.Bool("configs-only"), c.Bool("resume"), c.Bool("skip-check-parts-columns"), version, c.Int("command-id")) }, Flags: append(cliapp.Flags, cli.StringFlag{ Name: "table, tables, t", Hidden: false, - Usage: "create and upload backup only matched with table name patterns, separated by comma, allow ? and * as wildcard", + Usage: "Create and upload backup only matched with table name patterns, separated by comma, allow ? and * as wildcard", }, cli.StringSliceFlag{ Name: "partitions", Hidden: false, - Usage: "create and upload backup only for selected partition names, separated by comma\n" + - "if PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + - "if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + - "if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + - "values depends on field types in your table, use single quote for String and Date/DateTime related types\n" + - "look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", + Usage: "Create and upload backup only for selected partition names, separated by comma\n" + + "If PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + + "If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + + "If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + + "If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:*\n" + + "Values depends on field types in your table, use single quotes for String and Date/DateTime related types\n" + + "Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", }, cli.StringFlag{ Name: "diff-from", Hidden: false, - Usage: "local backup name which used to upload current backup as incremental", + Usage: "Local backup name which used to upload current backup as incremental", }, cli.StringFlag{ Name: "diff-from-remote", Hidden: false, - Usage: "remote backup name which used to upload current backup as incremental", + Usage: "Remote backup name which used to upload current backup as incremental", }, cli.BoolFlag{ Name: "schema, s", @@ -218,7 +236,12 @@ func main() { cli.BoolFlag{ Name: "skip-check-parts-columns", Hidden: false, - Usage: "skip check system.parts_columns to disallow backup inconsistent column types for data parts", + Usage: "Skip check system.parts_columns to disallow backup inconsistent column types for data parts", + }, + cli.BoolFlag{ + Name: "delete, delete-source, delete-local", + Hidden: false, + Usage: "explicitly delete local backup during upload", }, ), }, @@ -228,18 +251,18 @@ func main() { UsageText: "clickhouse-backup upload [-t, --tables=.
] [--partitions=] [-s, --schema] [--diff-from=] [--diff-from-remote=] [--resumable] ", Action: func(c *cli.Context) error { b := backup.NewBackuper(config.GetConfigFromCli(c)) - return b.Upload(c.Args().First(), c.String("diff-from"), c.String("diff-from-remote"), c.String("t"), c.StringSlice("partitions"), c.Bool("s"), c.Bool("resume"), c.Int("command-id")) + return b.Upload(c.Args().First(), c.Bool("delete-source"), c.String("diff-from"), c.String("diff-from-remote"), c.String("t"), c.StringSlice("partitions"), c.Bool("s"), c.Bool("resume"), version, c.Int("command-id")) }, Flags: append(cliapp.Flags, cli.StringFlag{ Name: "diff-from", Hidden: false, - Usage: "local backup name which used to upload current backup as incremental", + Usage: "Local backup name which used to upload current backup as incremental", }, cli.StringFlag{ Name: "diff-from-remote", Hidden: false, - Usage: "remote backup name which used to upload current backup as incremental", + Usage: "Remote backup name which used to upload current backup as incremental", }, cli.StringFlag{ Name: "table, tables, t", @@ -250,11 +273,12 @@ func main() { Name: "partitions", Hidden: false, Usage: "Upload backup only for selected partition names, separated by comma\n" + - "if PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + - "if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + - "if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + - "values depends on field types in your table, use single quote for String and Date/DateTime related types\n" + - "look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", + "If PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + + "If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + + "If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + + "If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:*\n" + + "Values depends on field types in your table, use single quotes for String and Date/DateTime related types\n" + + "Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", }, cli.BoolFlag{ Name: "schema, s", @@ -266,6 +290,11 @@ func main() { Hidden: false, Usage: "Save intermediate upload state and resume upload if backup exists on remote storage, ignored with 'remote_storage: custom' or 'use_embedded_backup_restore: true'", }, + cli.BoolFlag{ + Name: "delete, delete-source, delete-local", + Hidden: false, + Usage: "explicitly delete local backup during upload", + }, ), }, { @@ -284,7 +313,7 @@ func main() { UsageText: "clickhouse-backup download [-t, --tables=.
] [--partitions=] [-s, --schema] [--resumable] ", Action: func(c *cli.Context) error { b := backup.NewBackuper(config.GetConfigFromCli(c)) - return b.Download(c.Args().First(), c.String("t"), c.StringSlice("partitions"), c.Bool("s"), c.Bool("resume"), c.Int("command-id")) + return b.Download(c.Args().First(), c.String("t"), c.StringSlice("partitions"), c.Bool("s"), c.Bool("resume"), version, c.Int("command-id")) }, Flags: append(cliapp.Flags, cli.StringFlag{ @@ -296,11 +325,12 @@ func main() { Name: "partitions", Hidden: false, Usage: "Download backup data only for selected partition names, separated by comma\n" + - "if PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + - "if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + - "if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + - "values depends on field types in your table, use single quote for String and Date/DateTime related types\n" + - "look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", + "If PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + + "If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + + "If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + + "If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:*\n" + + "Values depends on field types in your table, use single quotes for String and Date/DateTime related types\n" + + "Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", }, cli.BoolFlag{ Name: "schema, s", @@ -317,10 +347,10 @@ func main() { { Name: "restore", Usage: "Create schema and restore data from backup", - UsageText: "clickhouse-backup restore [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--partitions=] [-s, --schema] [-d, --data] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] ", + UsageText: "clickhouse-backup restore [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--tm, --restore-table-mapping=:[,<...>]] [--partitions=] [-s, --schema] [-d, --data] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] ", Action: func(c *cli.Context) error { b := backup.NewBackuper(config.GetConfigFromCli(c)) - return b.Restore(c.Args().First(), c.String("t"), c.StringSlice("restore-database-mapping"), c.StringSlice("partitions"), c.Bool("s"), c.Bool("d"), c.Bool("rm"), c.Bool("ignore-dependencies"), c.Bool("rbac"), c.Bool("rbac-only"), c.Bool("configs"), c.Bool("configs-only"), c.Int("command-id")) + return b.Restore(c.Args().First(), c.String("t"), c.StringSlice("restore-database-mapping"), c.StringSlice("restore-table-mapping"), c.StringSlice("partitions"), c.Bool("schema"), c.Bool("data"), c.Bool("drop"), c.Bool("ignore-dependencies"), c.Bool("rbac"), c.Bool("rbac-only"), c.Bool("configs"), c.Bool("configs-only"), version, c.Int("command-id")) }, Flags: append(cliapp.Flags, cli.StringFlag{ @@ -333,15 +363,21 @@ func main() { Usage: "Define the rule to restore data. For the database not defined in this struct, the program will not deal with it.", Hidden: false, }, + cli.StringSliceFlag{ + Name: "restore-table-mapping, tm", + Usage: "Define the rule to restore data. For the table not defined in this struct, the program will not deal with it.", + Hidden: false, + }, cli.StringSliceFlag{ Name: "partitions", Hidden: false, Usage: "Restore backup only for selected partition names, separated by comma\n" + - "if PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + - "if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + - "if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + - "values depends on field types in your table, use single quote for String and Date/DateTime related types\n" + - "look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", + "If PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + + "If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + + "If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + + "If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:*\n" + + "Values depends on field types in your table, use single quotes for String and Date/DateTime related types\n" + + "Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", }, cli.BoolFlag{ Name: "schema, s", @@ -388,10 +424,10 @@ func main() { { Name: "restore_remote", Usage: "Download and restore", - UsageText: "clickhouse-backup restore_remote [--schema] [--data] [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--partitions=] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] [--skip-rbac] [--skip-configs] [--resumable] ", + UsageText: "clickhouse-backup restore_remote [--schema] [--data] [-t, --tables=.
] [-m, --restore-database-mapping=:[,<...>]] [--tm, --restore-table-mapping=:[,<...>]] [--partitions=] [--rm, --drop] [-i, --ignore-dependencies] [--rbac] [--configs] [--skip-rbac] [--skip-configs] [--resumable] ", Action: func(c *cli.Context) error { b := backup.NewBackuper(config.GetConfigFromCli(c)) - return b.RestoreFromRemote(c.Args().First(), c.String("t"), c.StringSlice("restore-database-mapping"), c.StringSlice("partitions"), c.Bool("s"), c.Bool("d"), c.Bool("rm"), c.Bool("i"), c.Bool("rbac"), c.Bool("rbac-only"), c.Bool("configs"), c.Bool("configs-only"), c.Bool("resume"), c.Int("command-id")) + return b.RestoreFromRemote(c.Args().First(), c.String("t"), c.StringSlice("restore-database-mapping"), c.StringSlice("restore-table-mapping"), c.StringSlice("partitions"), c.Bool("s"), c.Bool("d"), c.Bool("rm"), c.Bool("i"), c.Bool("rbac"), c.Bool("rbac-only"), c.Bool("configs"), c.Bool("configs-only"), c.Bool("resume"), version, c.Int("command-id")) }, Flags: append(cliapp.Flags, cli.StringFlag{ @@ -404,15 +440,21 @@ func main() { Usage: "Define the rule to restore data. For the database not defined in this struct, the program will not deal with it.", Hidden: false, }, + cli.StringSliceFlag{ + Name: "restore-table-mapping, tm", + Usage: "Define the rule to restore data. For the database not defined in this struct, the program will not deal with it.", + Hidden: false, + }, cli.StringSliceFlag{ Name: "partitions", Hidden: false, Usage: "Download and restore backup only for selected partition names, separated by comma\n" + - "if PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + - "if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + - "if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + - "values depends on field types in your table, use single quote for String and Date/DateTime related types\n" + - "look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", + "If PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + + "If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + + "If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + + "If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:*\n" + + "Values depends on field types in your table, use single quotes for String and Date/DateTime related types\n" + + "Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", }, cli.BoolFlag{ Name: "schema, s", @@ -547,12 +589,13 @@ func main() { cli.StringSliceFlag{ Name: "partitions", Hidden: false, - Usage: "partition names, separated by comma\n" + - "if PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + - "if PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + - "if PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + - "values depends on field types in your table, use single quote for String and Date/DateTime related types\n" + - "look to system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", + Usage: "Partitions names, separated by comma\n" + + "If PARTITION BY clause returns numeric not hashed values for `partition_id` field in system.parts table, then use --partitions=partition_id1,partition_id2 format\n" + + "If PARTITION BY clause returns hashed string values, then use --partitions=('non_numeric_field_value_for_part1'),('non_numeric_field_value_for_part2') format\n" + + "If PARTITION BY clause returns tuple with multiple fields, then use --partitions=(numeric_value1,'string_value1','date_or_datetime_value'),(...) format\n" + + "If you need different partitions for different tables, then use --partitions=db.table1:part1,part2 --partitions=db.table?:*\n" + + "Values depends on field types in your table, use single quotes for String and Date/DateTime related types\n" + + "Look at the system.parts partition and partition_id fields for details https://clickhouse.com/docs/en/operations/system-tables/parts/", }, cli.BoolFlag{ Name: "schema, s", @@ -572,7 +615,7 @@ func main() { cli.BoolFlag{ Name: "skip-check-parts-columns", Hidden: false, - Usage: "skip check system.parts_columns to disallow backup inconsistent column types for data parts", + Usage: "Skip check system.parts_columns to disallow backup inconsistent column types for data parts", }, ), }, @@ -585,7 +628,7 @@ func main() { Flags: append(cliapp.Flags, cli.BoolFlag{ Name: "watch", - Usage: "run watch go-routine for 'create_remote' + 'delete local', after API server startup", + Usage: "Run watch go-routine for 'create_remote' + 'delete local', after API server startup", Hidden: false, }, cli.StringFlag{ diff --git a/generate_manual.sh b/generate_manual.sh index 983d00d3..5108f3f5 100644 --- a/generate_manual.sh +++ b/generate_manual.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# make clean build-race -CLICKHOUSE_BACKUP_BIN=${CLICKHOUSE_BACKUP_BIN:-./clickhouse-backup/clickhouse-backup-race} +CLICKHOUSE_BACKUP_BIN=${CLICKHOUSE_BACKUP_BIN:-build/linux/$(dpkg --print-architecture)/clickhouse-backup} +make clean ${CLICKHOUSE_BACKUP_BIN} cmds=( tables create @@ -23,4 +23,4 @@ for cmd in ${cmds[@]}; do echo '```' "${CLICKHOUSE_BACKUP_BIN}" help "${cmd}" echo '```' -done \ No newline at end of file +done diff --git a/go.mod b/go.mod index ebed3f4d..a28788a4 100644 --- a/go.mod +++ b/go.mod @@ -1,135 +1,144 @@ -module github.com/Altinity/clickhouse-backup +module github.com/Altinity/clickhouse-backup/v2 require ( - cloud.google.com/go/storage v1.31.0 + cloud.google.com/go/storage v1.42.0 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Azure/go-autorest/autorest v0.11.29 - github.com/Azure/go-autorest/autorest/adal v0.9.23 - github.com/ClickHouse/clickhouse-go/v2 v2.13.0 - github.com/antchfx/xmlquery v1.3.17 - github.com/apex/log v1.9.0 - github.com/aws/aws-sdk-go-v2 v1.20.1 - github.com/aws/aws-sdk-go-v2/config v1.18.33 - github.com/aws/aws-sdk-go-v2/credentials v1.13.32 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.77 - github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 - github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 - github.com/aws/smithy-go v1.14.1 + github.com/Azure/go-autorest/autorest/adal v0.9.24 + // wrong INSERT syntax, wait when resolve https://github.com/ClickHouse/clickhouse-go/issues/1345 + github.com/ClickHouse/clickhouse-go/v2 v2.23.2 + github.com/antchfx/xmlquery v1.4.1 + github.com/aws/aws-sdk-go-v2 v1.30.1 + github.com/aws/aws-sdk-go-v2/config v1.27.23 + github.com/aws/aws-sdk-go-v2/credentials v1.17.23 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.3 + github.com/aws/aws-sdk-go-v2/service/s3 v1.57.1 + github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 + github.com/aws/smithy-go v1.20.3 github.com/djherbis/buffer v1.2.0 github.com/djherbis/nio/v3 v3.0.1 - github.com/eapache/go-resiliency v1.3.0 + github.com/eapache/go-resiliency v1.6.0 github.com/go-zookeeper/zk v1.0.3 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 - github.com/google/uuid v1.3.0 - github.com/gorilla/mux v1.8.0 + github.com/google/uuid v1.6.0 + github.com/gorilla/mux v1.8.1 github.com/jlaffaye/ftp v0.2.0 github.com/jolestar/go-commons-pool/v2 v2.1.2 github.com/kelseyhightower/envconfig v1.4.0 - github.com/klauspost/compress v1.16.7 + github.com/klauspost/compress v1.17.9 github.com/mattn/go-shellwords v1.0.12 github.com/mholt/archiver/v4 v4.0.0-alpha.8 - github.com/otiai10/copy v1.12.0 + github.com/otiai10/copy v1.14.0 github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.6 - github.com/prometheus/client_golang v1.16.0 - github.com/rs/zerolog v1.33.0 - github.com/stretchr/testify v1.8.4 - github.com/tencentyun/cos-go-sdk-v5 v0.7.42 - github.com/urfave/cli v1.22.14 + github.com/prometheus/client_golang v1.19.1 + github.com/puzpuzpuz/xsync v1.5.2 + github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285 + github.com/stretchr/testify v1.9.0 + github.com/tencentyun/cos-go-sdk-v5 v0.7.52 + github.com/urfave/cli v1.22.15 + github.com/xyproto/gionice v1.3.0 github.com/yargevad/filepathx v1.0.0 - golang.org/x/crypto v0.12.0 - golang.org/x/mod v0.12.0 - golang.org/x/sync v0.3.0 - google.golang.org/api v0.136.0 - gopkg.in/cheggaaa/pb.v1 v1.0.28 + golang.org/x/crypto v0.24.0 + golang.org/x/mod v0.18.0 + golang.org/x/sync v0.7.0 + google.golang.org/api v0.187.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - cloud.google.com/go v0.110.7 // indirect - cloud.google.com/go/compute v1.23.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.2 // indirect + github.com/apex/log v1.9.0 + github.com/rs/zerolog v1.33.0 + golang.org/x/text v0.16.0 +) + +require ( + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/auth v0.6.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect + cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/iam v1.1.8 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/ClickHouse/ch-go v0.58.0 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect - github.com/antchfx/xpath v1.2.4 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 // indirect + github.com/ClickHouse/ch-go v0.61.5 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect + github.com/antchfx/xpath v1.3.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bodgit/plumbing v1.3.0 // indirect - github.com/bodgit/sevenzip v1.4.3 // indirect - github.com/bodgit/windows v1.0.1 // indirect + github.com/bodgit/plumbing v1.2.0 // indirect + github.com/bodgit/sevenzip v1.3.0 // indirect + github.com/bodgit/windows v1.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/clbanning/mxj v1.8.4 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/connesc/cipherio v0.2.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect + github.com/dsnet/compress v0.0.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-faster/city v1.0.1 // indirect - github.com/go-faster/errors v0.6.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/go-querystring v1.1.0 // indirect - github.com/google/s2a-go v0.1.5 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect - github.com/googleapis/gax-go/v2 v2.12.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/google/go-querystring v1.0.0 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/pgzip v1.2.6 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect github.com/kr/fs v0.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-ieproxy v0.0.11 // indirect + github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/mozillazg/go-httpheader v0.4.0 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect - github.com/paulmach/orb v0.10.0 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect - github.com/rivo/uniseg v0.4.4 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect - github.com/shopspring/decimal v1.3.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.11 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel v1.16.0 // indirect - go.opentelemetry.io/otel/trace v1.16.0 // indirect - go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/text v0.12.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230807174057-1744710a1577 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230807174057-1744710a1577 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect + go4.org v0.0.0-20200411211856-f5505b9728dd // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/grpc v1.64.1 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) -go 1.21 +go 1.22 diff --git a/go.sum b/go.sum index c9d442b3..d271024e 100644 --- a/go.sum +++ b/go.sum @@ -7,265 +7,40 @@ cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTj cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= -cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= -cloud.google.com/go/accessapproval v1.7.1 h1:/5YjNhR6lzCvmJZAnByYkfEgWjfAKwYP6nkuTk6nKFE= -cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= -cloud.google.com/go/accesscontextmanager v1.8.1 h1:WIAt9lW9AXtqw/bnvrEUaE8VG/7bAAeMzRCBGMkc4+w= -cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= -cloud.google.com/go/aiplatform v1.48.0 h1:M5davZWCTzE043rJCn+ZLW6hSxfG1KAx4vJTtas2/ec= -cloud.google.com/go/aiplatform v1.48.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= -cloud.google.com/go/analytics v0.21.3 h1:TFBC1ZAqX9/jL56GEXdLrVe5vT3I22bDVWyDwZX4IEg= -cloud.google.com/go/analytics v0.21.3/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= -cloud.google.com/go/apigateway v1.6.1 h1:aBSwCQPcp9rZ0zVEUeJbR623palnqtvxJlUyvzsKGQc= -cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= -cloud.google.com/go/apigeeconnect v1.6.1 h1:6u/jj0P2c3Mcm+H9qLsXI7gYcTiG9ueyQL3n6vCmFJM= -cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= -cloud.google.com/go/apigeeregistry v0.7.1 h1:hgq0ANLDx7t2FDZDJQrCMtCtddR/pjCqVuvQWGrQbXw= -cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= -cloud.google.com/go/appengine v1.8.1 h1:J+aaUZ6IbTpBegXbmEsh8qZZy864ZVnOoWyfa1XSNbI= -cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= -cloud.google.com/go/area120 v0.8.1 h1:wiOq3KDpdqXmaHzvZwKdpoM+3lDcqsI2Lwhyac7stss= -cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= -cloud.google.com/go/artifactregistry v1.14.1 h1:k6hNqab2CubhWlGcSzunJ7kfxC7UzpAfQ1UPb9PDCKI= -cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= -cloud.google.com/go/asset v1.14.1 h1:vlHdznX70eYW4V1y1PxocvF6tEwxJTTarwIGwOhFF3U= -cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= -cloud.google.com/go/assuredworkloads v1.11.1 h1:yaO0kwS+SnhVSTF7BqTyVGt3DTocI6Jqo+S3hHmCwNk= -cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= -cloud.google.com/go/automl v1.13.1 h1:iP9iQurb0qbz+YOOMfKSEjhONA/WcoOIjt6/m+6pIgo= -cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= -cloud.google.com/go/baremetalsolution v1.1.1 h1:0Ge9PQAy6cZ1tRrkc44UVgYV15nw2TVnzJzYsMHXF+E= -cloud.google.com/go/baremetalsolution v1.1.1/go.mod h1:D1AV6xwOksJMV4OSlWHtWuFNZZYujJknMAP4Qa27QIA= -cloud.google.com/go/batch v1.3.1 h1:uE0Q//W7FOGPjf7nuPiP0zoE8wOT3ngoIO2HIet0ilY= -cloud.google.com/go/batch v1.3.1/go.mod h1:VguXeQKXIYaeeIYbuozUmBR13AfL4SJP7IltNPS+A4A= -cloud.google.com/go/beyondcorp v1.0.0 h1:VPg+fZXULQjs8LiMeWdLaB5oe8G9sEoZ0I0j6IMiG1Q= -cloud.google.com/go/beyondcorp v1.0.0/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38= +cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.53.0 h1:K3wLbjbnSlxhuG5q4pntHv5AEbQM1QqHKGYgwFIqOTg= -cloud.google.com/go/bigquery v1.53.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= -cloud.google.com/go/billing v1.16.0 h1:1iktEAIZ2uA6KpebC235zi/rCXDdDYQ0bTXTNetSL80= -cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= -cloud.google.com/go/binaryauthorization v1.6.1 h1:cAkOhf1ic92zEN4U1zRoSupTmwmxHfklcp1X7CCBKvE= -cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= -cloud.google.com/go/certificatemanager v1.7.1 h1:uKsohpE0hiobx1Eak9jNcPCznwfB6gvyQCcS28Ah9E8= -cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= -cloud.google.com/go/channel v1.16.0 h1:dqRkK2k7Ll/HHeYGxv18RrfhozNxuTJRkspW0iaFZoY= -cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= -cloud.google.com/go/cloudbuild v1.13.0 h1:YBbAWcvE4x6xPWTyS+OU4eiUpz5rCS3VCM/aqmfddPA= -cloud.google.com/go/cloudbuild v1.13.0/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= -cloud.google.com/go/clouddms v1.6.1 h1:rjR1nV6oVf2aNNB7B5uz1PDIlBjlOiBgR+q5n7bbB7M= -cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= -cloud.google.com/go/cloudtasks v1.12.1 h1:cMh9Q6dkvh+Ry5LAPbD/U2aw6KAqdiU6FttwhbTo69w= -cloud.google.com/go/cloudtasks v1.12.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/contactcenterinsights v1.10.0 h1:YR2aPedGVQPpFBZXJnPkqRj8M//8veIZZH5ZvICoXnI= -cloud.google.com/go/contactcenterinsights v1.10.0/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= -cloud.google.com/go/container v1.24.0 h1:N51t/cgQJFqDD/W7Mb+IvmAPHrf8AbPx7Bb7aF4lROE= -cloud.google.com/go/container v1.24.0/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= -cloud.google.com/go/containeranalysis v0.10.1 h1:SM/ibWHWp4TYyJMwrILtcBtYKObyupwOVeceI9pNblw= -cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= -cloud.google.com/go/datacatalog v1.16.0 h1:qVeQcw1Cz93/cGu2E7TYUPh8Lz5dn5Ws2siIuQ17Vng= -cloud.google.com/go/datacatalog v1.16.0/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= -cloud.google.com/go/dataflow v0.9.1 h1:VzG2tqsk/HbmOtq/XSfdF4cBvUWRK+S+oL9k4eWkENQ= -cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= -cloud.google.com/go/dataform v0.8.1 h1:xcWso0hKOoxeW72AjBSIp/UfkvpqHNzzS0/oygHlcqY= -cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= -cloud.google.com/go/datafusion v1.7.1 h1:eX9CZoyhKQW6g1Xj7+RONeDj1mV8KQDKEB9KLELX9/8= -cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= -cloud.google.com/go/datalabeling v0.8.1 h1:zxsCD/BLKXhNuRssen8lVXChUj8VxF3ofN06JfdWOXw= -cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= -cloud.google.com/go/dataplex v1.9.0 h1:yoBWuuUZklYp7nx26evIhzq8+i/nvKYuZr1jka9EqLs= -cloud.google.com/go/dataplex v1.9.0/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= -cloud.google.com/go/dataproc/v2 v2.0.1 h1:4OpSiPMMGV3XmtPqskBU/RwYpj3yMFjtMLj/exi425Q= -cloud.google.com/go/dataproc/v2 v2.0.1/go.mod h1:7Ez3KRHdFGcfY7GcevBbvozX+zyWGcwLJvvAMwCaoZ4= -cloud.google.com/go/dataqna v0.8.1 h1:ITpUJep04hC9V7C+gcK390HO++xesQFSUJ7S4nSnF3U= -cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.13.0 h1:ktbC66bOQB3HJPQe8qNI1/aiQ77PMu7hD4mzE6uxe3w= -cloud.google.com/go/datastore v1.13.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= -cloud.google.com/go/datastream v1.10.0 h1:ra/+jMv36zTAGPfi8TRne1hXme+UsKtdcK4j6bnqQiw= -cloud.google.com/go/datastream v1.10.0/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= -cloud.google.com/go/deploy v1.13.0 h1:A+w/xpWgz99EYzB6e31gMGAI/P5jTZ2UO7veQK5jQ8o= -cloud.google.com/go/deploy v1.13.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= -cloud.google.com/go/dialogflow v1.40.0 h1:sCJbaXt6ogSbxWQnERKAzos57f02PP6WkGbOZvXUdwc= -cloud.google.com/go/dialogflow v1.40.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= -cloud.google.com/go/dlp v1.10.1 h1:tF3wsJ2QulRhRLWPzWVkeDz3FkOGVoMl6cmDUHtfYxw= -cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= -cloud.google.com/go/documentai v1.22.0 h1:dW8ex9yb3oT9s1yD2+yLcU8Zq15AquRZ+wd0U+TkxFw= -cloud.google.com/go/documentai v1.22.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= -cloud.google.com/go/domains v0.9.1 h1:rqz6KY7mEg7Zs/69U6m6LMbB7PxFDWmT3QWNXIqhHm0= -cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= -cloud.google.com/go/edgecontainer v1.1.1 h1:zhHWnLzg6AqzE+I3gzJqiIwHfjEBhWctNQEzqb+FaRo= -cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= -cloud.google.com/go/errorreporting v0.3.0 h1:kj1XEWMu8P0qlLhm3FwcaFsUvXChV/OraZwA70trRR0= -cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= -cloud.google.com/go/essentialcontacts v1.6.2 h1:OEJ0MLXXCW/tX1fkxzEZOsv/wRfyFsvDVNaHWBAvoV0= -cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= -cloud.google.com/go/eventarc v1.13.0 h1:xIP3XZi0Xawx8DEfh++mE2lrIi5kQmCr/KcWhJ1q0J4= -cloud.google.com/go/eventarc v1.13.0/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= -cloud.google.com/go/filestore v1.7.1 h1:Eiz8xZzMJc5ppBWkuaod/PUdUZGCFR8ku0uS+Ah2fRw= -cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= -cloud.google.com/go/firestore v1.12.0 h1:aeEA/N7DW7+l2u5jtkO8I0qv0D95YwjggD8kUHrTHO4= -cloud.google.com/go/firestore v1.12.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= -cloud.google.com/go/functions v1.15.1 h1:LtAyqvO1TFmNLcROzHZhV0agEJfBi+zfMZsF4RT/a7U= -cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= -cloud.google.com/go/gkebackup v1.3.0 h1:lgyrpdhtJKV7l1GM15YFt+OCyHMxsQZuSydyNmS0Pxo= -cloud.google.com/go/gkebackup v1.3.0/go.mod h1:vUDOu++N0U5qs4IhG1pcOnD1Mac79xWy6GoBFlWCWBU= -cloud.google.com/go/gkeconnect v0.8.1 h1:a1ckRvVznnuvDWESM2zZDzSVFvggeBaVY5+BVB8tbT0= -cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= -cloud.google.com/go/gkehub v0.14.1 h1:2BLSb8i+Co1P05IYCKATXy5yaaIw/ZqGvVSBTLdzCQo= -cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= -cloud.google.com/go/gkemulticloud v1.0.0 h1:MluqhtPVZReoriP5+adGIw+ij/RIeRik8KApCW2WMTw= -cloud.google.com/go/gkemulticloud v1.0.0/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= -cloud.google.com/go/gsuiteaddons v1.6.1 h1:mi9jxZpzVjLQibTS/XfPZvl+Jr6D5Bs8pGqUjllRb00= -cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= -cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= -cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= -cloud.google.com/go/iap v1.8.1 h1:X1tcp+EoJ/LGX6cUPt3W2D4H2Kbqq0pLAsldnsCjLlE= -cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= -cloud.google.com/go/ids v1.4.1 h1:khXYmSoDDhWGEVxHl4c4IgbwSRR+qE/L4hzP3vaU9Hc= -cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= -cloud.google.com/go/iot v1.7.1 h1:yrH0OSmicD5bqGBoMlWG8UltzdLkYzNUwNVUVz7OT54= -cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= -cloud.google.com/go/kms v1.15.0 h1:xYl5WEaSekKYN5gGRyhjvZKM22GVBBCzegGNVPy+aIs= -cloud.google.com/go/kms v1.15.0/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= -cloud.google.com/go/language v1.10.1 h1:3MXeGEv8AlX+O2LyV4pO4NGpodanc26AmXwOuipEym0= -cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= -cloud.google.com/go/lifesciences v0.9.1 h1:axkANGx1wiBXHiPcJZAE+TDjjYoJRIDzbHC/WYllCBU= -cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= -cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I= -cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= -cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= -cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= -cloud.google.com/go/managedidentities v1.6.1 h1:2/qZuOeLgUHorSdxSQGtnOu9xQkBn37+j+oZQv/KHJY= -cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= -cloud.google.com/go/maps v1.4.0 h1:PdfgpBLhAoSzZrQXP+/zBc78fIPLZSJp5y8+qSMn2UU= -cloud.google.com/go/maps v1.4.0/go.mod h1:6mWTUv+WhnOwAgjVsSW2QPPECmW+s3PcRyOa9vgG/5s= -cloud.google.com/go/mediatranslation v0.8.1 h1:50cF7c1l3BanfKrpnTCaTvhf+Fo6kdF21DG0byG7gYU= -cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= -cloud.google.com/go/memcache v1.10.1 h1:7lkLsF0QF+Mre0O/NvkD9Q5utUNwtzvIYjrOLOs0HO0= -cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= -cloud.google.com/go/metastore v1.12.0 h1:+9DsxUOHvsqvC0ylrRc/JwzbXJaaBpfIK3tX0Lx8Tcc= -cloud.google.com/go/metastore v1.12.0/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= -cloud.google.com/go/monitoring v1.15.1 h1:65JhLMd+JiYnXr6j5Z63dUYCuOg770p8a/VC+gil/58= -cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= -cloud.google.com/go/networkconnectivity v1.12.1 h1:LnrYM6lBEeTq+9f2lR4DjBhv31EROSAQi/P5W4Q0AEc= -cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= -cloud.google.com/go/networkmanagement v1.8.0 h1:/3xP37eMxnyvkfLrsm1nv1b2FbMMSAEAOlECTvoeCq4= -cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= -cloud.google.com/go/networksecurity v0.9.1 h1:TBLEkMp3AE+6IV/wbIGRNTxnqLXHCTEQWoxRVC18TzY= -cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ= -cloud.google.com/go/notebooks v1.9.1 h1:CUqMNEtv4EHFnbogV+yGHQH5iAQLmijOx191innpOcs= -cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8= -cloud.google.com/go/optimization v1.4.1 h1:pEwOAmO00mxdbesCRSsfj8Sd4rKY9kBrYW7Vd3Pq7cA= -cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk= -cloud.google.com/go/orchestration v1.8.1 h1:KmN18kE/xa1n91cM5jhCh7s1/UfIguSCisw7nTMUzgE= -cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8= -cloud.google.com/go/orgpolicy v1.11.1 h1:I/7dHICQkNwym9erHqmlb50LRU588NPCvkfIY0Bx9jI= -cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE= -cloud.google.com/go/osconfig v1.12.1 h1:dgyEHdfqML6cUW6/MkihNdTVc0INQst0qSE8Ou1ub9c= -cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= -cloud.google.com/go/oslogin v1.10.1 h1:LdSuG3xBYu2Sgr3jTUULL1XCl5QBx6xwzGqzoDUw1j0= -cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= -cloud.google.com/go/phishingprotection v0.8.1 h1:aK/lNmSd1vtbft/vLe2g7edXK72sIQbqr2QyrZN/iME= -cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= -cloud.google.com/go/policytroubleshooter v1.8.0 h1:XTMHy31yFmXgQg57CB3w9YQX8US7irxDX0Fl0VwlZyY= -cloud.google.com/go/policytroubleshooter v1.8.0/go.mod h1:tmn5Ir5EToWe384EuboTcVQT7nTag2+DuH3uHmKd1HU= -cloud.google.com/go/privatecatalog v0.9.1 h1:B/18xGo+E0EMS9LOEQ0zXz7F2asMgmVgTYGSI89MHOA= -cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.33.0 h1:6SPCPvWav64tj0sVX/+npCBKhUi/UjJehy9op/V3p2g= -cloud.google.com/go/pubsub v1.33.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= -cloud.google.com/go/pubsublite v1.8.1 h1:pX+idpWMIH30/K7c0epN6V703xpIcMXWRjKJsz0tYGY= -cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= -cloud.google.com/go/recaptchaenterprise/v2 v2.7.2 h1:IGkbudobsTXAwmkEYOzPCQPApUCsN4Gbq3ndGVhHQpI= -cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU= -cloud.google.com/go/recommendationengine v0.8.1 h1:nMr1OEVHuDambRn+/y4RmNAmnR/pXCuHtH0Y4tCgGRQ= -cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE= -cloud.google.com/go/recommender v1.10.1 h1:UKp94UH5/Lv2WXSQe9+FttqV07x/2p1hFTMMYVFtilg= -cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA= -cloud.google.com/go/redis v1.13.1 h1:YrjQnCC7ydk+k30op7DSjSHw1yAYhqYXFcOq1bSXRYA= -cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= -cloud.google.com/go/resourcemanager v1.9.1 h1:QIAMfndPOHR6yTmMUB0ZN+HSeRmPjR/21Smq5/xwghI= -cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8= -cloud.google.com/go/resourcesettings v1.6.1 h1:Fdyq418U69LhvNPFdlEO29w+DRRjwDA4/pFamm4ksAg= -cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw= -cloud.google.com/go/retail v1.14.1 h1:gYBrb9u/Hc5s5lUTFXX1Vsbc/9BEvgtioY6ZKaK0DK8= -cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE= -cloud.google.com/go/run v1.2.0 h1:kHeIG8q+N6Zv0nDkBjSOYfK2eWqa5FnaiDPH/7/HirE= -cloud.google.com/go/run v1.2.0/go.mod h1:36V1IlDzQ0XxbQjUx6IYbw8H3TJnWvhii963WW3B/bo= -cloud.google.com/go/scheduler v1.10.1 h1:yoZbZR8880KgPGLmACOMCiY2tPk+iX4V/dkxqTirlz8= -cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo= -cloud.google.com/go/secretmanager v1.11.1 h1:cLTCwAjFh9fKvU6F13Y4L9vPcx9yiWPyWXE4+zkuEQs= -cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= -cloud.google.com/go/security v1.15.1 h1:jR3itwycg/TgGA0uIgTItcVhA55hKWiNJxaNNpQJaZE= -cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= -cloud.google.com/go/securitycenter v1.23.0 h1:XOGJ9OpnDtqg8izd7gYk/XUhj8ytjIalyjjsR6oyG0M= -cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= -cloud.google.com/go/servicedirectory v1.11.0 h1:pBWpjCFVGWkzVTkqN3TBBIqNSoSHY86/6RL0soSQ4z8= -cloud.google.com/go/servicedirectory v1.11.0/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= -cloud.google.com/go/shell v1.7.1 h1:aHbwH9LSqs4r2rbay9f6fKEls61TAjT63jSyglsw7sI= -cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= -cloud.google.com/go/spanner v1.47.0 h1:aqiMP8dhsEXgn9K5EZBWxPG7dxIiyM2VaikqeU4iteg= -cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= -cloud.google.com/go/speech v1.19.0 h1:MCagaq8ObV2tr1kZJcJYgXYbIn8Ai5rp42tyGYw9rls= -cloud.google.com/go/speech v1.19.0/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI= -cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0= -cloud.google.com/go/storagetransfer v1.10.0 h1:+ZLkeXx0K0Pk5XdDmG0MnUVqIR18lllsihU/yq39I8Q= -cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= -cloud.google.com/go/talent v1.6.2 h1:j46ZgD6N2YdpFPux9mc7OAf4YK3tiBCsbLKc8rQx+bU= -cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24= -cloud.google.com/go/texttospeech v1.7.1 h1:S/pR/GZT9p15R7Y2dk2OXD/3AufTct/NSxT4a7nxByw= -cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= -cloud.google.com/go/tpu v1.6.1 h1:kQf1jgPY04UJBYYjNUO+3GrZtIb57MfGAW2bwgLbR3A= -cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= -cloud.google.com/go/trace v1.10.1 h1:EwGdOLCNfYOOPtgqo+D2sDLZmRCEO1AagRTJCU6ztdg= -cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= -cloud.google.com/go/translate v1.8.2 h1:PQHamiOzlehqLBJMnM72lXk/OsMQewZB12BKJ8zXrU0= -cloud.google.com/go/translate v1.8.2/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= -cloud.google.com/go/video v1.19.0 h1:BRyyS+wU+Do6VOXnb8WfPr42ZXti9hzmLKLUCkggeK4= -cloud.google.com/go/video v1.19.0/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= -cloud.google.com/go/videointelligence v1.11.1 h1:MBMWnkQ78GQnRz5lfdTAbBq/8QMCF3wahgtHh3s/J+k= -cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= -cloud.google.com/go/vision/v2 v2.7.2 h1:ccK6/YgPfGHR/CyESz1mvIbsht5Y2xRsWCPqmTNydEw= -cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= -cloud.google.com/go/vmmigration v1.7.1 h1:gnjIclgqbEMc+cF5IJuPxp53wjBIlqZ8h9hE8Rkwp7A= -cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= -cloud.google.com/go/vmwareengine v1.0.0 h1:qsJ0CPlOQu/3MFBGklu752v3AkD+Pdu091UmXJ+EjTA= -cloud.google.com/go/vmwareengine v1.0.0/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= -cloud.google.com/go/vpcaccess v1.7.1 h1:ram0GzjNWElmbxXMIzeOZUkQ9J8ZAahD6V8ilPGqX0Y= -cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= -cloud.google.com/go/webrisk v1.9.1 h1:Ssy3MkOMOnyRV5H2bkMQ13Umv7CwB/kugo3qkAX83Fk= -cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= -cloud.google.com/go/websecurityscanner v1.6.1 h1:CfEF/vZ+xXyAR3zC9iaC/QRdf1MEgS20r5UR17Q4gOg= -cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= -cloud.google.com/go/workflows v1.11.1 h1:2akeQ/PgtRhrNuD/n1WvJd5zb7YyuDZrlOanBj2ihPg= -cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= +cloud.google.com/go/storage v1.42.0 h1:4QtGpplCVt1wz6g5o1ifXd656P5z+yNgzdw1tVfp0cU= +cloud.google.com/go/storage v1.42.0/go.mod h1:HjMXRFq65pGKFn6hxj6x3HCyR41uSB72Z0SO/Vn6JFQ= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= -github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= -github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= +github.com/Azure/go-autorest/autorest/adal v0.9.24 h1:BHZfgGsGwdkHDyZdtQRQk1WeUdW0m2WPAwuHZwUi5i4= +github.com/Azure/go-autorest/autorest/adal v0.9.24/go.mod h1:7T1+g0PYFmACYW5LlG2fcoPiPlFHjClyRGL7dRlP5c8= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= @@ -276,211 +51,132 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ClickHouse/ch-go v0.58.0 h1:SwCH/WWJnwHnQdehiQsmraC13+uMpkexPR61c2Jj5Qo= -github.com/ClickHouse/ch-go v0.58.0/go.mod h1:P0YtetDdxICIC7IuRM9Qv/TMi83W0mRgBVQ9wcpt+4Y= -github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= -github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= -github.com/ClickHouse/clickhouse-go/v2 v2.13.0 h1:oP1OlTQIbQKKLnqLzyDhiyNFvN3pbOtM+e/3qdexG9k= -github.com/ClickHouse/clickhouse-go/v2 v2.13.0/go.mod h1:xyL0De2K54/n+HGsdtPuyYJq76wefafaHfGUXTDEq/0= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/hcsshim v0.9.6 h1:VwnDOgLeoi2du6dAznfmspNqTiwczvjv4K7NxuY9jsY= -github.com/Microsoft/hcsshim v0.9.6/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409 h1:DTQ/38ao/CfXsrK0cSAL+h4R/u0VVvfWLZEOlLwEROI= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= +github.com/ClickHouse/clickhouse-go/v2 v2.23.2 h1:+DAKPMnxLS7pduQZsrJc8OhdLS2L9MfDEJ2TS+hpYDM= +github.com/ClickHouse/clickhouse-go/v2 v2.23.2/go.mod h1:aNap51J1OM3yxQJRgM+AlP/MPkGBCL8A74uQThoQhR0= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= -github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU= -github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/antchfx/xmlquery v1.3.17 h1:d0qWjPp/D+vtRw7ivCwT5ApH/3CkQU8JOeo3245PpTk= -github.com/antchfx/xmlquery v1.3.17/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA= -github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= -github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= -github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/antchfx/xmlquery v1.4.1 h1:YgpSwbeWvLp557YFTi8E3z6t6/hYjmFEtiEKbDfEbl0= +github.com/antchfx/xmlquery v1.4.1/go.mod h1:lKezcT8ELGt8kW5L+ckFMTbgdR61/odpPgDv8Gvi1fI= +github.com/antchfx/xpath v1.3.1 h1:PNbFuUqHwWl0xRjvUPjJ95Agbmdj2uzzIwmQKgu4oCk= +github.com/antchfx/xpath v1.3.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= -github.com/apex/logs v1.0.0 h1:adOwhOTeXzZTnVuEK13wuJNBFutP0sOfutRS8NY+G6A= github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= -github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a h1:2KLQMJ8msqoPHIPDufkxVcoTtcmE5+1sL9950m4R9Pk= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= -github.com/aphistic/sweet v0.2.0 h1:I4z+fAUqvKfvZV/CHi5dV0QuwbmIvYYFDjG0Ss5QpAs= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= -github.com/aws/aws-sdk-go v1.20.6 h1:kmy4Gvdlyez1fV4kw5RYxZzWKVyuHZHgPWeU/YvRsV4= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v1.20.1 h1:rZBf5DWr7YGrnlTK4kgDQGn1ltqOg5orCYb/UhOFZkg= -github.com/aws/aws-sdk-go-v2 v1.20.1/go.mod h1:NU06lETsFm8fUC6ZjhgDpVBcGZTFQ6XM+LZWZxMI4ac= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 h1:lN6L3LrYHeZ6xCxaIYtoWCx4GMLk4nRknsh29OMSqHY= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12/go.mod h1:TDCkEAkMTXxTs0oLBGBKpBZbk3NLh8EvAfF0Q3x8/0c= -github.com/aws/aws-sdk-go-v2/config v1.18.33 h1:JKcw5SFxFW/rpM4mOPjv0VQ11E2kxW13F3exWOy7VZU= -github.com/aws/aws-sdk-go-v2/config v1.18.33/go.mod h1:hXO/l9pgY3K5oZJldamP0pbZHdPqqk+4/maa7DSD3cA= -github.com/aws/aws-sdk-go-v2/credentials v1.13.32 h1:lIH1eKPcCY1ylR4B6PkBGRWMHO3aVenOKJHWiS4/G2w= -github.com/aws/aws-sdk-go-v2/credentials v1.13.32/go.mod h1:lL8U3v/Y79YRG69WlAho0OHIKUXCyFvSXaIvfo81sls= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 h1:DK/9C+UN/X+1+Wm8pqaDksQr2tSLzq+8X1/rI/ZxKEQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8/go.mod h1:ce7BgLQfYr5hQFdy67oX2svto3ufGtm6oBvmsHScI1Q= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.77 h1:oWSNL9oQy+do911sXpJyIc2J7RiUrbm9BecyaGy1wHo= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.77/go.mod h1:xvOdc97VpScJqB10YAI8r/cKuU7d9Ls/as03KROO2qY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 h1:c8ed/T9T2K5I+h/JzmF5tpI46+OODQ74dzmdo+QnaMg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38/go.mod h1:qggunOChCMu9ZF/UkAfhTz25+U2rLVb3ya0Ua6TTfCA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 h1:hNeAAymUY5gu11WrrmFb3CVIp9Dar9hbo44yzzcQpzA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32/go.mod h1:0ZXSqrty4FtQ7p8TEuRde/SZm9X05KT18LAUlR40Ln0= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 h1:fc0ukRAiP1syoSGZYu+DaE+FulSYhTiJ8WpVu5jElU4= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39/go.mod h1:WLAW8PT7+JhjZfLSWe7WEJaJu0GNo0cKc2Zyo003RBs= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 h1:vUh7dBFNS3oFCtVv6CiYKh5hP9ls8+kIpKLeFruIBLk= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1/go.mod h1:sFMeinkhj/SZKQM8BxtvNtSPjJEo0Xrz+w3g2e4FSKI= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 h1:iV/W5OMBys+66OeXJi/7xIRrKZNsu0ylsLGu+6nbmQE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13/go.mod h1:ReJb6xYmtGyu9KoFtRreWegbN9dZqvZIIv4vWnhcsyI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 h1:QviNkc+vGSuEHx8P+pVNKOdWLXBPIwMFv7p0fphgE4U= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33/go.mod h1:fABTUmOrAgAalG2i9WJpjBvlnk7UK8YmnYaxN+Q2CwE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 h1:dGAseBFEYxth10V23b5e2mAS+tX7oVbfYHD6dnDdAsg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32/go.mod h1:4jwAWKEkCR0anWk5+1RbfSg1R5Gzld7NLiuaq5bTR/Y= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 h1:PT6PBCycRwhpEW5hJnRiceCeoWJ+r3bdgXtV+VKG7Pk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1/go.mod h1:TqoxCLwT2nrxrBGA+z7t6OWM7LBkgRckK3gOjYE+7JA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2 h1:v346f1h8sUBKXnEbrv43L37MTBlFHyKXQPIZHNAaghA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2/go.mod h1:cwCATiyNrXK9P2FsWdZ89g9mpsYv2rhk0UA/KByl5fY= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 h1:A2RlEMo4SJSwbNoUUgkxTAEMduAy/8wG3eB2b2lP4gY= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.2/go.mod h1:ju+nNXUunfIFamXUIZQiICjnO/TPlOmWcYhZcSy7xaE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 h1:OJELEgyaT2kmaBGZ+myyZbTTLobfe3ox3FSh5eYK9Qs= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2/go.mod h1:ubDBBaDFs1GHijSOTi8ljppML15GLG0HxhILtbjNNYQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 h1:ympg1+Lnq33XLhcK/xTG4yZHPs1Oyxu+6DEWbl7qOzA= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.2/go.mod h1:FQ/DQcOfESELfJi5ED+IPPAjI5xC6nxtSolVVB773jM= -github.com/aws/smithy-go v1.14.1 h1:EFKMUmH/iHMqLiwoEDx2rRjRQpI1YCn5jTysoaDujFs= -github.com/aws/smithy-go v1.14.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo= +github.com/aws/aws-sdk-go-v2 v1.30.1 h1:4y/5Dvfrhd1MxRDD77SrfsDaj8kUkkljU7XE83NPV+o= +github.com/aws/aws-sdk-go-v2 v1.30.1/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM= +github.com/aws/aws-sdk-go-v2/config v1.27.23 h1:Cr/gJEa9NAS7CDAjbnB7tHYb3aLZI2gVggfmSAasDac= +github.com/aws/aws-sdk-go-v2/config v1.27.23/go.mod h1:WMMYHqLCFu5LH05mFOF5tsq1PGEMfKbu083VKqLCd0o= +github.com/aws/aws-sdk-go-v2/credentials v1.17.23 h1:G1CfmLVoO2TdQ8z9dW+JBc/r8+MqyPQhXCafNZcXVZo= +github.com/aws/aws-sdk-go-v2/credentials v1.17.23/go.mod h1:V/DvSURn6kKgcuKEk4qwSwb/fZ2d++FFARtWSbXnLqY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.3 h1:J2mHCzCeDQNfBOas73ARi4/CsLm0wYpQ3Itll8dPDBQ= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.3/go.mod h1:6rYGWnaLHD+WRF4E709VW+HEEJPKZbNdjHgq9osFXuE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13 h1:THZJJ6TU/FOiM7DZFnisYV9d49oxXWUzsVIMTuf3VNU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.13/go.mod h1:VISUTg6n+uBaYIWPBaIG0jk7mbBxm7DUqBtU2cUDDWI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15 h1:2jyRZ9rVIMisyQRnhSS/SqlckveoxXneIumECVFP91Y= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.15/go.mod h1:bDRG3m382v1KJBk1cKz7wIajg87/61EiiymEyfLvAe0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 h1:I9zMeF107l0rJrpnHpjEiiTSCKYAIw8mALiXcPsGBiA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15/go.mod h1:9xWJ3Q/S6Ojusz1UIkfycgD1mGirJfLLKqq3LPT7WN8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13 h1:Eq2THzHt6P41mpjS2sUzz/3dJYFRqdWZ+vQaEMm98EM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.13/go.mod h1:FgwTca6puegxgCInYwGjmd4tB9195Dd6LCuA+8MjpWw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.57.1 h1:aHPtNY87GZ214N4rShgIo+5JQz7ICrJ50i17JbueUTw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.57.1/go.mod h1:hdV0NTYd0RwV4FvNKhKUNbPLZoq9CTr/lke+3I7aCAI= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 h1:p1GahKIjyMDZtiKoIn0/jAj/TkMzfzndDv5+zi2Mhgc= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.1/go.mod h1:/vWdhoIoYA5hYoPZ6fm7Sv4d8701PiG5VKe8/pPJL60= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.1 h1:lCEv9f8f+zJ8kcFeAjRZsekLd/x5SAm96Cva+VbUdo8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.1/go.mod h1:xyFHA4zGxgYkdD73VeezHt3vSKEG9EmFnGwoKlP00u4= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMwlW+Wqm2Ppsfp4nQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ= +github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= +github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= -github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= -github.com/bodgit/sevenzip v1.4.3 h1:46Rb9vCYdpceC1U+GIR0bS3hP2/Xv8coKFDeLJySV/A= -github.com/bodgit/sevenzip v1.4.3/go.mod h1:F8n3+0CwbdxqmNy3wFeOAtanza02Ur66AGfs/hbYblI= -github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= -github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/bodgit/plumbing v1.2.0 h1:gg4haxoKphLjml+tgnecR4yLBV5zo4HAZGCtAh3xCzM= +github.com/bodgit/plumbing v1.2.0/go.mod h1:b9TeRi7Hvc6Y05rjm8VML3+47n4XTZPtQ/5ghqic2n8= +github.com/bodgit/sevenzip v1.3.0 h1:1ljgELgtHqvgIp8W8kgeEGHIWP4ch3xGI8uOBZgLVKY= +github.com/bodgit/sevenzip v1.3.0/go.mod h1:omwNcgZTEooWM8gA/IJ2Nk/+ZQ94+GsytRzOJJ8FBlM= +github.com/bodgit/windows v1.0.0 h1:rLQ/XjsleZvx4fR1tB/UxQrK+SJ2OFHzfPjLWWOhDIA= +github.com/bodgit/windows v1.0.0/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= -github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= -github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe h1:QQ3GSy+MqSHxm/d8nCtnAiZdYFd45cYZPs8vOOIYKfk= -github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEycw= github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA= -github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= -github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= -github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns= -github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ= github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE= github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4= github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= -github.com/dmarkham/enumer v1.5.8 h1:fIF11F9l5jyD++YYvxcSH5WgHfeaSGPaN/T4kOQ4qEM= -github.com/dmarkham/enumer v1.5.8/go.mod h1:d10o8R3t/gROm2p3BXqTkMt2+HMuxEmWCXzorAruYak= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.22+incompatible h1:6jX4yB+NtcbldT90k7vBSaWJDB3i+zkVJT9BEK8kQkk= -github.com/docker/docker v20.10.22+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= -github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= -github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 h1:tFh1tRc4CA31yP6qDcu+Trax5wW5GuMxvkIba07qVLY= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= -github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-resiliency v1.6.0 h1:CqGDTLtpwuWKn6Nj3uNUdflaq+/kIPsg0gfNzHton30= +github.com/eapache/go-resiliency v1.6.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f h1:7T++XKzy4xg7PKy+bM+Sa9/oe1OC88yz2hXQUISoXfA= -github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8= -github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= -github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI= -github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72 h1:b+9H1GAsx5RsjvDFLoS5zkNBzIQMuVKUYQDmxU3N5XE= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= -github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-zookeeper/zk v1.0.3 h1:7M2kwOsc//9VeeFiPtf+uSJlVpU66x9Ba5+8XK7/TDg= github.com/go-zookeeper/zk v1.0.3/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= -github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -489,8 +185,9 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3 h1:GV+pQPG/EUUbkh47niozDcADz6go/dUwhVzdUQHIVRw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -501,17 +198,14 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -521,61 +215,45 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-pkcs11 v0.2.0 h1:5meDPB26aJ98f+K9G21f0AqZwo/S5BJMJh8nuhMbdsI= -github.com/google/go-pkcs11 v0.2.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12 h1:TgXhFz35pKlZuUz1pNlOKk1UCSXPpuUIc144Wd7SxCA= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.5 h1:8IYp3w9nysqv3JH+NJgXJzGbDHzLOTj43BmSkp+O7qg= -github.com/google/s2a-go v0.1.5/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/grafov/m3u8 v0.12.0/go.mod h1:nqzOkfBiZJENr52zTVd/Dcl03yzphIMbJqkXGu+u080= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jackc/puddle/v2 v2.2.0 h1:RdcDk92EJBuBS55nQMMYFXTxwstHug4jkhT5pq8VxPk= -github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -586,116 +264,64 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/jolestar/go-commons-pool/v2 v2.1.2 h1:E+XGo58F23t7HtZiC/W6jzO2Ux2IccSH/yx4nD+J1CM= github.com/jolestar/go-commons-pool/v2 v2.1.2/go.mod h1:r4NYccrkS5UqP1YQI1COyTZ9UjPJAAGTUxzcsK1kqhY= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= -github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= -github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= -github.com/mattn/go-ieproxy v0.0.11 h1:MQ/5BuGSgDAHZOJe6YY80IF2UVCfGkwfo6AeD7HtHYo= -github.com/mattn/go-ieproxy v0.0.11/go.mod h1:/NsJd+kxZBmjMc5hrJCKMbP57B84rvq9BiDRbtO9AS0= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM= github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615 h1:/mD+ABZyXD39BzJI2XyRJlqdZG11gXFo0SSynL+OFeU= -github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= -github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs= -github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0= -github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= -github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w= -github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= -github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= -github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= -github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= -github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= -github.com/pascaldekloe/name v1.0.1 h1:9lnXOHeqeHHnWLbKfH6X98+4+ETVqFqxN09UXSjcMb0= -github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM= -github.com/paulmach/orb v0.10.0 h1:guVYVqzxHE/CQ1KpfGO077TR0ATHSNjp4s6XGLn3W9s= -github.com/paulmach/orb v0.10.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= -github.com/paulmach/protoscan v0.2.1 h1:rM0FpcTjUMvPUNk2BhPJrreDKetq43ChnL+x1sRg8O8= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -703,118 +329,81 @@ github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/puzpuzpuz/xsync v1.5.2 h1:yRAP4wqSOZG+/4pxJ08fPTwrfL0IzE/LKQ/cw509qGY= +github.com/puzpuzpuz/xsync v1.5.2/go.mod h1:K98BYhX3k1dQ2M63t1YNVDanbwUPmBCAhNmVrrxfiGg= +github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285 h1:d54EL9l+XteliUfUCGsEwwuk65dmmxX85VXF+9T6+50= +github.com/ricochet2200/go-disk-usage/du v0.0.0-20210707232629-ac9918953285/go.mod h1:fxIDly1xtudczrZeOOlfaUvd2OPb2qZAPuWdU2BsBTk= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= -github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= -github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 h1:hp2CYQUINdZMHdvTdXtPOY2ainKl4IoMcpAXEf2xj3Q= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= -github.com/smartystreets/gunit v1.0.0 h1:RyPDUFcJbvtXlhJPk7v+wnxZRY2EUokhEYl2EJOPToI= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563 h1:2VDxTtn9dAqI2DnnvB9fXpPE4DblOmquyzmN2zxTD8A= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= -github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563 h1:FoX+MK4vHThvPO6FbP5q98zD8S3n+d5+DbtK7skl++c= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0= -github.com/tencentyun/cos-go-sdk-v5 v0.7.42 h1:Up1704BJjI5orycXKjpVpvuOInt9GC5pqY4knyE9Uds= -github.com/tencentyun/cos-go-sdk-v5 v0.7.42/go.mod h1:LUFnaqRmGk6pEHOaRmdn2dCZR2j0cSsM5xowWFPTPao= -github.com/testcontainers/testcontainers-go v0.14.0 h1:h0D5GaYG9mhOWr2qHdEKDXpkce/VlvaYOCzTRi6UBi8= -github.com/testcontainers/testcontainers-go v0.14.0/go.mod h1:hSRGJ1G8Q5Bw2gXgPulJOLlEBaYJHeBSOkQM5JLG+JQ= +github.com/tencentyun/cos-go-sdk-v5 v0.7.52 h1:gCCQBWGNs8Bgx0VSsld5vjgi2U+MiMwjbzy7sDMyKiA= +github.com/tencentyun/cos-go-sdk-v5 v0.7.52/go.mod h1:UN+VdbCl1hg+kKi5RXqZgaP+Boqfmk+D04GRc4XFk70= github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= -github.com/tj/go-buffer v1.1.0 h1:Lo2OsPHlIxXF24zApe15AbK3bJLAOvkkxEA6Ux4c47M= github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= -github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2 h1:eGaGNxrtoZf/mBURsnNQKDR7u50Klgcf2eFDQEnc8Bc= github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= -github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b h1:m74UWYy+HBs+jMFR9mdZU6shPewugMyH5+GV6LNgW8w= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= -github.com/tj/go-spin v1.1.0 h1:lhdWZsvImxvZ3q1C5OIB7d72DuOwP4O2NdBg9PyzNds= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= -github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= -github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= -github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= -github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= -github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/urfave/cli v1.22.15 h1:nuqt+pdC/KqswQKhETJjo7pvn/k4xMUxgW6liI7XpnM= +github.com/urfave/cli v1.22.15/go.mod h1:wSan1hmo5zeyLGBjRJbzRTNk8gwoYa2B9n4q9dmRIc0= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= -github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= -github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +github.com/xyproto/gionice v1.3.0 h1:v0X22Iduy+lplGT735pKRc6c550vvXYXvgXy0ZkA+qM= +github.com/xyproto/gionice v1.3.0/go.mod h1:bXnNfrv26yrsvmIXmTPztA+/MJ6wDjKv4l5can8P0cw= github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= -github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -822,26 +411,20 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= -go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= -go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= -go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= -go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= -go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= -go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= -go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU= +go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -851,13 +434,13 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -865,10 +448,8 @@ golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm0 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd h1:zkO/Lhoka23X63N9OSzpSeROEUQ5ODw47tM3YWjygbs= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -878,10 +459,8 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -889,8 +468,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -906,8 +486,8 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -917,15 +497,16 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -935,8 +516,9 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -952,29 +534,30 @@ golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -982,14 +565,16 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -1017,14 +602,13 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1033,15 +617,13 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.136.0 h1:e/6enzUE1s4tGPa6Q3ZYShKTtvRc+1Jq0rrafhppmOs= -google.golang.org/api v0.136.0/go.mod h1:XtJfF+V2zgUxelOn5Zs3kECtluMxneJG8ZxUTlLNTPA= +google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo= +google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1055,16 +637,13 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230807174057-1744710a1577 h1:Tyk/35yqszRCvaragTn5NnkY6IiKk/XvHzEWepo71N0= -google.golang.org/genproto v0.0.0-20230807174057-1744710a1577/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230807174057-1744710a1577 h1:xv8KoglAClYGkprUSmDTKaILtzfD8XzG9NYVXMprjKo= -google.golang.org/genproto/googleapis/api v0.0.0-20230807174057-1744710a1577/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20230807174057-1744710a1577 h1:ZX0eQu2J+jOO87sq8fQG8J/Nfp7D7BhHpixIE5EYK/k= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20230807174057-1744710a1577/go.mod h1:NjCQG/D8JandXxM57PZbAJL1DCNL6EypA0vPPwfsc7c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 h1:wukfNtZmZUurLN/atp2hiIeTKn7QJWIQdHzqmsOnAOk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d h1:PksQg4dV6Sem3/HkBX+Ltq8T0ke0PKIRBNBatoDTVls= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1073,12 +652,9 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1089,26 +665,19 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= -gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -1120,11 +689,7 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/pkg/backup/backup_shard.go b/pkg/backup/backup_shard.go index aa762c59..9aca4fb4 100644 --- a/pkg/backup/backup_shard.go +++ b/pkg/backup/backup_shard.go @@ -96,7 +96,7 @@ func fnvShardReplicaFromString(str string, activeReplicas []string) (string, err h := fnv.New32a() if _, err := h.Write([]byte(str)); err != nil { - log.Fatal().Stack().Msgf("can't write %s", str) + log.Fatal().Stack().Msgf("can't write %s to fnv.New32a", str) } i := h.Sum32() % uint32(len(activeReplicas)) return activeReplicas[i], nil diff --git a/pkg/backup/backuper.go b/pkg/backup/backuper.go index e182288b..0160d49f 100644 --- a/pkg/backup/backuper.go +++ b/pkg/backup/backuper.go @@ -4,12 +4,17 @@ import ( "context" "errors" "fmt" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "net/url" + "os" "path" + "strings" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/resumable" - "github.com/Altinity/clickhouse-backup/pkg/storage" + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/resumable" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage" + "github.com/rs/zerolog/log" ) const DirectoryFormat = "directory" @@ -65,7 +70,7 @@ func WithBackupSharder(s backupSharder) BackuperOpt { } } -func (b *Backuper) init(ctx context.Context, disks []clickhouse.Disk, backupName string) error { +func (b *Backuper) initDisksPathsAndBackupDestination(ctx context.Context, disks []clickhouse.Disk, backupName string) error { var err error if disks == nil { disks, err = b.ch.GetDisks(ctx, true) @@ -84,6 +89,9 @@ func (b *Backuper) init(ctx context.Context, disks []clickhouse.Disk, backupName b.EmbeddedBackupDataPath = disk.Path } } + if b.cfg.ClickHouse.UseEmbeddedBackupRestore && b.EmbeddedBackupDataPath == "" { + b.EmbeddedBackupDataPath = b.DefaultDataPath + } b.DiskToPathMap = diskMap if b.cfg.General.RemoteStorage != "none" && b.cfg.General.RemoteStorage != "custom" { b.dst, err = storage.NewBackupDestination(ctx, b.cfg, b.ch, true, backupName) @@ -147,3 +155,224 @@ func (b *Backuper) populateBackupShardField(ctx context.Context, tables []clickh } return nil } + +func (b *Backuper) isDiskTypeObject(diskType string) bool { + return diskType == "s3" || diskType == "azure_blob_storage" || diskType == "azure" +} + +func (b *Backuper) isDiskTypeEncryptedObject(disk clickhouse.Disk, disks []clickhouse.Disk) bool { + if disk.Type != "encrypted" { + return false + } + underlyingIdx := -1 + underlyingPath := "" + for i, d := range disks { + if d.Name != disk.Name && strings.HasPrefix(disk.Path, d.Path) && b.isDiskTypeObject(d.Type) { + if d.Path > underlyingPath { + underlyingIdx = i + underlyingPath = d.Path + } + } + } + return underlyingIdx >= 0 +} + +func (b *Backuper) getEmbeddedBackupDefaultSettings(version int) []string { + settings := []string{} + if (b.cfg.General.RemoteStorage == "s3" || b.cfg.General.RemoteStorage == "gcs") && version >= 23007000 { + settings = append(settings, "allow_s3_native_copy=1") + if err := b.ch.Query("SET s3_request_timeout_ms=600000"); err != nil { + log.Fatal().Msgf("SET s3_request_timeout_ms=600000 error: %v", err) + } + + } + if (b.cfg.General.RemoteStorage == "s3" || b.cfg.General.RemoteStorage == "gcs") && version >= 23011000 { + if err := b.ch.Query("SET s3_use_adaptive_timeouts=0"); err != nil { + log.Fatal().Msgf("SET s3_use_adaptive_timeouts=0 error: %v", err) + } + } + if b.cfg.General.RemoteStorage == "azblob" && version >= 24005000 { + settings = append(settings, "allow_azure_native_copy=1") + } + return settings +} + +func (b *Backuper) getEmbeddedBackupLocation(ctx context.Context, backupName string) (string, error) { + if b.cfg.ClickHouse.EmbeddedBackupDisk != "" { + return fmt.Sprintf("Disk('%s','%s')", b.cfg.ClickHouse.EmbeddedBackupDisk, backupName), nil + } + + if err := b.applyMacrosToObjectDiskPath(ctx); err != nil { + return "", err + } + if b.cfg.General.RemoteStorage == "s3" { + s3Endpoint, err := b.ch.ApplyMacros(ctx, b.buildEmbeddedLocationS3()) + if err != nil { + return "", err + } + if b.cfg.S3.AccessKey != "" { + return fmt.Sprintf("S3('%s/%s','%s','%s')", s3Endpoint, backupName, b.cfg.S3.AccessKey, b.cfg.S3.SecretKey), nil + } + if os.Getenv("AWS_ACCESS_KEY_ID") != "" { + return fmt.Sprintf("S3('%s/%s','%s','%s')", s3Endpoint, backupName, os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY")), nil + } + return "", fmt.Errorf("provide s3->access_key and s3->secret_key in config to allow embedded backup without `clickhouse->embedded_backup_disk`") + } + if b.cfg.General.RemoteStorage == "gcs" { + gcsEndpoint, err := b.ch.ApplyMacros(ctx, b.buildEmbeddedLocationGCS()) + if err != nil { + return "", err + } + if b.cfg.GCS.EmbeddedAccessKey != "" { + return fmt.Sprintf("S3('%s/%s','%s','%s')", gcsEndpoint, backupName, b.cfg.GCS.EmbeddedAccessKey, b.cfg.GCS.EmbeddedSecretKey), nil + } + if os.Getenv("AWS_ACCESS_KEY_ID") != "" { + return fmt.Sprintf("S3('%s/%s','%s','%s')", gcsEndpoint, backupName, os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY")), nil + } + return "", fmt.Errorf("provide gcs->embedded_access_key and gcs->embedded_secret_key in config to allow embedded backup without `clickhouse->embedded_backup_disk`") + } + if b.cfg.General.RemoteStorage == "azblob" { + azblobEndpoint, err := b.ch.ApplyMacros(ctx, b.buildEmbeddedLocationAZBLOB()) + if err != nil { + return "", err + } + if b.cfg.AzureBlob.Container != "" { + return fmt.Sprintf("AzureBlobStorage('%s','%s','%s/%s')", azblobEndpoint, b.cfg.AzureBlob.Container, b.cfg.AzureBlob.ObjectDiskPath, backupName), nil + } + return "", fmt.Errorf("provide azblob->container and azblob->account_name, azblob->account_key in config to allow embedded backup without `clickhouse->embedded_backup_disk`") + } + return "", fmt.Errorf("empty clickhouse->embedded_backup_disk and invalid general->remote_storage: %s", b.cfg.General.RemoteStorage) +} + +func (b *Backuper) applyMacrosToObjectDiskPath(ctx context.Context) error { + var err error + if b.cfg.General.RemoteStorage == "s3" { + b.cfg.S3.ObjectDiskPath, err = b.ch.ApplyMacros(ctx, b.cfg.S3.ObjectDiskPath) + } else if b.cfg.General.RemoteStorage == "gcs" { + b.cfg.GCS.ObjectDiskPath, err = b.ch.ApplyMacros(ctx, b.cfg.GCS.ObjectDiskPath) + } else if b.cfg.General.RemoteStorage == "azblob" { + b.cfg.AzureBlob.ObjectDiskPath, err = b.ch.ApplyMacros(ctx, b.cfg.AzureBlob.ObjectDiskPath) + } + return err +} + +func (b *Backuper) buildEmbeddedLocationS3() string { + s3backupURL := url.URL{} + s3backupURL.Scheme = "https" + if strings.HasPrefix(b.cfg.S3.Endpoint, "http") { + newUrl, _ := s3backupURL.Parse(b.cfg.S3.Endpoint) + s3backupURL = *newUrl + s3backupURL.Path = path.Join(b.cfg.S3.Bucket, b.cfg.S3.ObjectDiskPath) + } else { + s3backupURL.Host = b.cfg.S3.Endpoint + s3backupURL.Path = path.Join(b.cfg.S3.Bucket, b.cfg.S3.ObjectDiskPath) + } + if b.cfg.S3.DisableSSL { + s3backupURL.Scheme = "http" + } + if s3backupURL.Host == "" && b.cfg.S3.Region != "" && b.cfg.S3.ForcePathStyle { + s3backupURL.Host = "s3." + b.cfg.S3.Region + ".amazonaws.com" + s3backupURL.Path = path.Join(b.cfg.S3.Bucket, b.cfg.S3.ObjectDiskPath) + } + if s3backupURL.Host == "" && b.cfg.S3.Bucket != "" && !b.cfg.S3.ForcePathStyle { + s3backupURL.Host = b.cfg.S3.Bucket + "." + "s3." + b.cfg.S3.Region + ".amazonaws.com" + s3backupURL.Path = b.cfg.S3.ObjectDiskPath + } + return s3backupURL.String() +} + +func (b *Backuper) buildEmbeddedLocationGCS() string { + gcsBackupURL := url.URL{} + gcsBackupURL.Scheme = "https" + if b.cfg.GCS.ForceHttp { + gcsBackupURL.Scheme = "http" + } + if b.cfg.GCS.Endpoint != "" { + if !strings.HasPrefix(b.cfg.GCS.Endpoint, "http") { + gcsBackupURL.Host = b.cfg.GCS.Endpoint + } else { + newUrl, _ := gcsBackupURL.Parse(b.cfg.GCS.Endpoint) + gcsBackupURL = *newUrl + } + } + if gcsBackupURL.Host == "" { + gcsBackupURL.Host = "storage.googleapis.com" + } + gcsBackupURL.Path = path.Join(b.cfg.GCS.Bucket, b.cfg.GCS.ObjectDiskPath) + return gcsBackupURL.String() +} + +func (b *Backuper) buildEmbeddedLocationAZBLOB() string { + azblobBackupURL := url.URL{} + azblobBackupURL.Scheme = b.cfg.AzureBlob.EndpointSchema + azblobBackupURL.Host = b.cfg.AzureBlob.EndpointSuffix + azblobBackupURL.Path = b.cfg.AzureBlob.AccountName + return fmt.Sprintf("DefaultEndpointsProtocol=%s;AccountName=%s;AccountKey=%s;BlobEndpoint=%s;", b.cfg.AzureBlob.EndpointSchema, b.cfg.AzureBlob.AccountName, b.cfg.AzureBlob.AccountKey, azblobBackupURL.String()) +} + +func (b *Backuper) getObjectDiskPath() (string, error) { + if b.cfg.General.RemoteStorage == "s3" { + return b.cfg.S3.ObjectDiskPath, nil + } else if b.cfg.General.RemoteStorage == "azblob" { + return b.cfg.AzureBlob.ObjectDiskPath, nil + } else if b.cfg.General.RemoteStorage == "gcs" { + return b.cfg.GCS.ObjectDiskPath, nil + } else { + return "", fmt.Errorf("cleanBackupObjectDisks: requesst object disks path but have unsupported remote_storage: %s", b.cfg.General.RemoteStorage) + } +} + +func (b *Backuper) getTablesDiffFromLocal(ctx context.Context, diffFrom string, tablePattern string) (tablesForUploadFromDiff map[metadata.TableTitle]metadata.TableMetadata, err error) { + tablesForUploadFromDiff = make(map[metadata.TableTitle]metadata.TableMetadata) + diffFromBackup, err := b.ReadBackupMetadataLocal(ctx, diffFrom) + if err != nil { + return nil, err + } + if len(diffFromBackup.Tables) != 0 { + metadataPath := path.Join(b.DefaultDataPath, "backup", diffFrom, "metadata") + // empty partitions, because we don't want filter + diffTablesList, _, err := b.getTableListByPatternLocal(ctx, metadataPath, tablePattern, false, []string{}) + if err != nil { + return nil, err + } + for _, t := range diffTablesList { + tablesForUploadFromDiff[metadata.TableTitle{ + Database: t.Database, + Table: t.Table, + }] = t + } + } + return tablesForUploadFromDiff, nil +} + +func (b *Backuper) getTablesDiffFromRemote(ctx context.Context, diffFromRemote string, tablePattern string) (tablesForUploadFromDiff map[metadata.TableTitle]metadata.TableMetadata, err error) { + tablesForUploadFromDiff = make(map[metadata.TableTitle]metadata.TableMetadata) + backupList, err := b.dst.BackupList(ctx, true, diffFromRemote) + if err != nil { + return nil, err + } + var diffRemoteMetadata *metadata.BackupMetadata + for _, backup := range backupList { + if backup.BackupName == diffFromRemote { + diffRemoteMetadata = &backup.BackupMetadata + break + } + } + if diffRemoteMetadata == nil { + return nil, fmt.Errorf("%s not found on remote storage", diffFromRemote) + } + + if len(diffRemoteMetadata.Tables) != 0 { + diffTablesList, err := getTableListByPatternRemote(ctx, b, diffRemoteMetadata, tablePattern, false) + if err != nil { + return nil, err + } + for _, t := range diffTablesList { + tablesForUploadFromDiff[metadata.TableTitle{ + Database: t.Database, + Table: t.Table, + }] = t + } + } + return tablesForUploadFromDiff, nil +} diff --git a/pkg/backup/backuper_test.go b/pkg/backup/backuper_test.go index c067ef79..694c7d17 100644 --- a/pkg/backup/backuper_test.go +++ b/pkg/backup/backuper_test.go @@ -6,8 +6,8 @@ import ( "reflect" "testing" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" ) type testVersioner struct { diff --git a/pkg/backup/create.go b/pkg/backup/create.go index 60a19a4e..2b9a3819 100644 --- a/pkg/backup/create.go +++ b/pkg/backup/create.go @@ -8,23 +8,29 @@ import ( "os" "path" "path/filepath" + "sort" "strings" + "sync" + "sync/atomic" "time" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/common" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/filesystemhelper" - "github.com/Altinity/clickhouse-backup/pkg/keeper" - "github.com/Altinity/clickhouse-backup/pkg/metadata" - "github.com/Altinity/clickhouse-backup/pkg/partition" - "github.com/Altinity/clickhouse-backup/pkg/status" - "github.com/Altinity/clickhouse-backup/pkg/storage" - "github.com/Altinity/clickhouse-backup/pkg/storage/object_disk" - "github.com/Altinity/clickhouse-backup/pkg/utils" "github.com/google/uuid" recursiveCopy "github.com/otiai10/copy" + "golang.org/x/sync/errgroup" + + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/common" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/filesystemhelper" + "github.com/Altinity/clickhouse-backup/v2/pkg/keeper" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "github.com/Altinity/clickhouse-backup/v2/pkg/partition" + "github.com/Altinity/clickhouse-backup/v2/pkg/status" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage/object_disk" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" "github.com/rs/zerolog/log" + ) const ( @@ -40,7 +46,6 @@ var ( type LocalBackup struct { metadata.BackupMetadata - Legacy bool Broken string } @@ -51,7 +56,7 @@ func NewBackupName() string { // CreateBackup - create new backup of all tables matched by tablePattern // If backupName is empty string will use default backup name -func (b *Backuper) CreateBackup(backupName, tablePattern string, partitions []string, schemaOnly, createRBAC, rbacOnly, createConfigs, configsOnly, skipCheckPartsColumns bool, version string, commandId int) error { +func (b *Backuper) CreateBackup(backupName, diffFromRemote, tablePattern string, partitions []string, schemaOnly, createRBAC, rbacOnly, createConfigs, configsOnly, skipCheckPartsColumns bool, backupVersion string, commandId int) error { ctx, cancel, err := status.Current.GetContextWithCancel(commandId) if err != nil { return err @@ -60,7 +65,6 @@ func (b *Backuper) CreateBackup(backupName, tablePattern string, partitions []st defer cancel() startBackup := time.Now() - doBackupData := !schemaOnly && !rbacOnly && !configsOnly if backupName == "" { backupName = NewBackupName() } @@ -73,6 +77,9 @@ func (b *Backuper) CreateBackup(backupName, tablePattern string, partitions []st if skipCheckPartsColumns && b.cfg.ClickHouse.CheckPartsColumns { b.cfg.ClickHouse.CheckPartsColumns = false } + if b.cfg.General.RBACBackupAlways { + createRBAC = true + } allDatabases, err := b.ch.GetDatabases(ctx, b.cfg, tablePattern) if err != nil { @@ -102,6 +109,14 @@ func (b *Backuper) CreateBackup(backupName, tablePattern string, partitions []st if err != nil { return err } + version, err := b.ch.GetVersion(ctx) + if err != nil { + return err + } + b.DefaultDataPath, err = b.ch.GetDefaultPath(disks) + if err != nil { + return err + } diskMap := make(map[string]string, len(disks)) diskTypes := make(map[string]string, len(disks)) @@ -110,13 +125,25 @@ func (b *Backuper) CreateBackup(backupName, tablePattern string, partitions []st diskTypes[disk.Name] = disk.Type } partitionsIdMap, partitionsNameList := partition.ConvertPartitionsToIdsMapAndNamesList(ctx, b.ch, tables, nil, partitions) - // create + doBackupData := !schemaOnly && !rbacOnly && !configsOnly + backupRBACSize, backupConfigSize, rbacAndConfigsErr := b.createRBACAndConfigsIfNecessary(ctx, backupName, createRBAC, rbacOnly, createConfigs, configsOnly, disks, diskMap) + if rbacAndConfigsErr != nil { + return rbacAndConfigsErr + } if b.cfg.ClickHouse.UseEmbeddedBackupRestore { - err = b.createBackupEmbedded(ctx, backupName, tablePattern, partitionsNameList, partitionsIdMap, schemaOnly, createRBAC, createConfigs, tables, allDatabases, allFunctions, disks, diskMap, diskTypes, startBackup, version) + err = b.createBackupEmbedded(ctx, backupName, diffFromRemote, doBackupData, schemaOnly, backupVersion, tablePattern, partitionsNameList, partitionsIdMap, tables, allDatabases, allFunctions, disks, diskMap, diskTypes, backupRBACSize, backupConfigSize, startBackup, version) } else { - err = b.createBackupLocal(ctx, backupName, partitionsIdMap, tables, doBackupData, schemaOnly, createRBAC, rbacOnly, createConfigs, configsOnly, version, disks, diskMap, diskTypes, allDatabases, allFunctions, startBackup) + err = b.createBackupLocal(ctx, backupName, diffFromRemote, doBackupData, schemaOnly, rbacOnly, configsOnly, backupVersion, partitionsIdMap, tables, tablePattern, disks, diskMap, diskTypes, allDatabases, allFunctions, backupRBACSize, backupConfigSize, startBackup, version) } if err != nil { + // delete local backup if can't create + if removeBackupErr := b.RemoveBackupLocal(ctx, backupName, disks); removeBackupErr != nil { + log.Error().Msgf("creating failed -> b.RemoveBackupLocal error: %v", removeBackupErr) + } + // fix corner cases after https://github.com/Altinity/clickhouse-backup/issues/379 + if cleanShadowErr := b.Clean(ctx); cleanShadowErr != nil { + log.Error().Msgf("creating failed -> b.Clean error: %v", cleanShadowErr) + } return err } @@ -127,18 +154,45 @@ func (b *Backuper) CreateBackup(backupName, tablePattern string, partitions []st return nil } -func (b *Backuper) createBackupLocal(ctx context.Context, backupName string, partitionsIdMap map[metadata.TableTitle]common.EmptyMap, tables []clickhouse.Table, doBackupData bool, schemaOnly bool, createRBAC, rbacOnly bool, createConfigs, configsOnly bool, version string, disks []clickhouse.Disk, diskMap, diskTypes map[string]string, allDatabases []clickhouse.Database, allFunctions []clickhouse.Function, startBackup time.Time) error { +func (b *Backuper) createRBACAndConfigsIfNecessary(ctx context.Context, backupName string, createRBAC bool, rbacOnly bool, createConfigs bool, configsOnly bool, disks []clickhouse.Disk, diskMap map[string]string) (uint64, uint64, error) { + backupRBACSize, backupConfigSize := uint64(0), uint64(0) + backupPath := path.Join(b.DefaultDataPath, "backup") + if b.cfg.ClickHouse.EmbeddedBackupDisk != "" { + backupPath = diskMap[b.cfg.ClickHouse.EmbeddedBackupDisk] + } + backupPath = path.Join(backupPath, backupName) + if createRBAC || rbacOnly { + var createRBACErr error + if backupRBACSize, createRBACErr = b.createBackupRBAC(ctx, backupPath, disks); createRBACErr != nil { + log.Fatal().Msgf("error during do RBAC backup: %v", createRBACErr) + } else { + log.Info().Str("size", utils.FormatBytes(backupRBACSize)).Msg("done createBackupRBAC") + } + } + if createConfigs || configsOnly { + var createConfigsErr error + if backupConfigSize, createConfigsErr = b.createBackupConfigs(ctx, backupPath); createConfigsErr != nil { + log.Fatal().Msgf("error during do CONFIG backup: %v", createConfigsErr) + } else { + log.Info().Str("size", utils.FormatBytes(backupConfigSize)).Msg("done createBackupConfigs") + } + } + if backupRBACSize > 0 || backupConfigSize > 0 { + if chownErr := filesystemhelper.Chown(backupPath, b.ch, disks, true); chownErr != nil { + return backupRBACSize, backupConfigSize, chownErr + } + } + return backupRBACSize, backupConfigSize, nil +} + +func (b *Backuper) createBackupLocal(ctx context.Context, backupName, diffFromRemote string, doBackupData, schemaOnly, rbacOnly, configsOnly bool, backupVersion string, partitionsIdMap map[metadata.TableTitle]common.EmptyMap, tables []clickhouse.Table, tablePattern string, disks []clickhouse.Disk, diskMap, diskTypes map[string]string, allDatabases []clickhouse.Database, allFunctions []clickhouse.Function, backupRBACSize, backupConfigSize uint64, startBackup time.Time, version int) error { // Create backup dir on all clickhouse disks for _, disk := range disks { if err := filesystemhelper.Mkdir(path.Join(disk.Path, "backup"), b.ch, disks); err != nil { return err } } - defaultPath, err := b.ch.GetDefaultPath(disks) - if err != nil { - return err - } - backupPath := path.Join(defaultPath, "backup", backupName) + backupPath := path.Join(b.DefaultDataPath, "backup", backupName) if _, err := os.Stat(path.Join(backupPath, "metadata.json")); err == nil || !os.IsNotExist(err) { return fmt.Errorf("'%s' medatata.json already exists", backupName) } @@ -148,55 +202,100 @@ func (b *Backuper) createBackupLocal(ctx context.Context, backupName string, par return err } } - var backupDataSize, backupMetadataSize uint64 + isObjectDiskContainsTables := false + for _, disk := range disks { + if b.isDiskTypeObject(disk.Type) || b.isDiskTypeEncryptedObject(disk, disks) { + for _, table := range tables { + sort.Slice(table.DataPaths, func(i, j int) bool { return len(table.DataPaths[i]) > len(table.DataPaths[j]) }) + for _, tableDataPath := range table.DataPaths { + if strings.HasPrefix(tableDataPath, disk.Path) { + isObjectDiskContainsTables = true + break + } + } + } + } + } - var tableMetas []metadata.TableTitle - for _, table := range tables { - select { - case <-ctx.Done(): - return ctx.Err() - default: - if table.Skip { - continue + var err error + // https://github.com/Altinity/clickhouse-backup/issues/910 + if isObjectDiskContainsTables { + if err = config.ValidateObjectDiskConfig(b.cfg); err != nil { + return err + } + } + + if isObjectDiskContainsTables || (diffFromRemote != "" && b.cfg.General.RemoteStorage != "custom") { + b.dst, err = storage.NewBackupDestination(ctx, b.cfg, b.ch, false, backupName) + if err != nil { + return err + } + if err = b.dst.Connect(ctx); err != nil { + return fmt.Errorf("can't connect to %s: %v", b.dst.Kind(), err) + } + defer func() { + if closeErr := b.dst.Close(ctx); closeErr != nil { + log.Warn().Msgf("can't close connection to %s: %v", b.dst.Kind(), closeErr) } - var realSize map[string]int64 + }() + } + var tablesDiffFromRemote map[metadata.TableTitle]metadata.TableMetadata + if diffFromRemote != "" && b.cfg.General.RemoteStorage != "custom" { + var diffFromRemoteErr error + tablesDiffFromRemote, diffFromRemoteErr = b.getTablesDiffFromRemote(ctx, diffFromRemote, tablePattern) + if diffFromRemoteErr != nil { + return fmt.Errorf("b.getTablesDiffFromRemote return error: %v", diffFromRemoteErr) + } + } + + var backupDataSize, backupObjectDiskSize, backupMetadataSize uint64 + var metaMutex sync.Mutex + createBackupWorkingGroup, createCtx := errgroup.WithContext(ctx) + createBackupWorkingGroup.SetLimit(max(b.cfg.ClickHouse.MaxConnections, 1)) + + var tableMetas []metadata.TableTitle + for tableIdx, tableItem := range tables { + //to avoid race condition + table := tableItem + if table.Skip { + continue + } + idx := tableIdx + createBackupWorkingGroup.Go(func() error { + logger := log.With().Str("table", fmt.Sprintf("%s.%s", table.Database, table.Name)).Logger() + var realSize, objectDiskSize map[string]int64 var disksToPartsMap map[string][]metadata.Part if doBackupData && table.BackupType == clickhouse.ShardBackupFull { - log.Debug().Msg("create data") + logger.Debug().Msg("create data") shadowBackupUUID := strings.ReplaceAll(uuid.New().String(), "-", "") - disksToPartsMap, realSize, err = b.AddTableToBackup(ctx, backupName, shadowBackupUUID, disks, &table, partitionsIdMap[metadata.TableTitle{Database: table.Database, Table: table.Name}]) - if err != nil { - log.Error().Msg(err.Error()) - if removeBackupErr := b.RemoveBackupLocal(ctx, backupName, disks); removeBackupErr != nil { - log.Error().Msg(removeBackupErr.Error()) - } - // fix corner cases after https://github.com/Altinity/clickhouse-backup/issues/379 - if cleanShadowErr := b.Clean(ctx); cleanShadowErr != nil { - log.Error().Msg(cleanShadowErr.Error()) - } - return err + var addTableToBackupErr error + disksToPartsMap, realSize, objectDiskSize, addTableToBackupErr = b.AddTableToLocalBackup(createCtx, backupName, tablesDiffFromRemote, shadowBackupUUID, disks, &table, partitionsIdMap[metadata.TableTitle{Database: table.Database, Table: table.Name}], version) + if addTableToBackupErr != nil { + logger.Error().Msgf("b.AddTableToLocalBackup error: %v", addTableToBackupErr) + return addTableToBackupErr } // more precise data size calculation for _, size := range realSize { - backupDataSize += uint64(size) + atomic.AddUint64(&backupDataSize, uint64(size)) + } + for _, size := range objectDiskSize { + atomic.AddUint64(&backupObjectDiskSize, uint64(size)) } } // https://github.com/Altinity/clickhouse-backup/issues/529 - log.Debug().Msg("get in progress mutations list") + logger.Debug().Msg("get in progress mutations list") inProgressMutations := make([]metadata.MutationMetadata, 0) if b.cfg.ClickHouse.BackupMutations && !schemaOnly && !rbacOnly && !configsOnly { - inProgressMutations, err = b.ch.GetInProgressMutations(ctx, table.Database, table.Name) - if err != nil { - log.Error().Msg(err.Error()) - if removeBackupErr := b.RemoveBackupLocal(ctx, backupName, disks); removeBackupErr != nil { - log.Error().Msg(removeBackupErr.Error()) - } - return err + var inProgressMutationsErr error + inProgressMutations, inProgressMutationsErr = b.ch.GetInProgressMutations(createCtx, table.Database, table.Name) + if inProgressMutationsErr != nil { + logger.Error().Msgf("b.ch.GetInProgressMutations error: %v", inProgressMutationsErr) + return inProgressMutationsErr } } - log.Debug().Msg("create metadata") + logger.Debug().Msg("create metadata") if schemaOnly || doBackupData { - metadataSize, err := b.createTableMetadata(path.Join(backupPath, "metadata"), metadata.TableMetadata{ + metadataSize, createTableMetadataErr := b.createTableMetadata(path.Join(backupPath, "metadata"), metadata.TableMetadata{ Table: table.Name, Database: table.Database, Query: table.CreateTableQuery, @@ -206,208 +305,309 @@ func (b *Backuper) createBackupLocal(ctx context.Context, backupName string, par Mutations: inProgressMutations, MetadataOnly: schemaOnly || table.BackupType == clickhouse.ShardBackupSchema, }, disks) - if err != nil { - if removeBackupErr := b.RemoveBackupLocal(ctx, backupName, disks); removeBackupErr != nil { - log.Error().Msg(removeBackupErr.Error()) - } - return err + if createTableMetadataErr != nil { + logger.Error().Msgf("b.createTableMetadata error: %v", createTableMetadataErr) + return createTableMetadataErr } - backupMetadataSize += metadataSize + atomic.AddUint64(&backupMetadataSize, metadataSize) + metaMutex.Lock() tableMetas = append(tableMetas, metadata.TableTitle{ Database: table.Database, Table: table.Name, }) + metaMutex.Unlock() } - log.Info().Str("database", table.Database).Str("table", table.Name).Msgf("done") - } + logger.Info().Str("progress", fmt.Sprintf("%d/%d", idx+1, len(tables))).Msg("done") + return nil + }) } - backupRBACSize, backupConfigSize := uint64(0), uint64(0) - - if createRBAC || rbacOnly { - if backupRBACSize, err = b.createBackupRBAC(ctx, backupPath, disks); err != nil { - log.Fatal().Msgf("error during do RBAC backup: %v", err) - } else { - log.Info().Str("size", utils.FormatBytes(backupRBACSize)).Msg("done createRBACBackup") - } - } - if createConfigs || configsOnly { - if backupConfigSize, err = b.createBackupConfigs(ctx, backupPath); err != nil { - log.Fatal().Msgf("error during do CONFIG backup: %v", err) - } else { - log.Info().Str("size", utils.FormatBytes(backupConfigSize)).Msg("done createBackupConfigs") - } + if wgWaitErr := createBackupWorkingGroup.Wait(); wgWaitErr != nil { + return fmt.Errorf("one of createBackupLocal go-routine return error: %v", wgWaitErr) } - backupMetaFile := path.Join(defaultPath, "backup", backupName, "metadata.json") - if err := b.createBackupMetadata(ctx, backupMetaFile, backupName, version, "regular", diskMap, diskTypes, disks, backupDataSize, backupMetadataSize, backupRBACSize, backupConfigSize, tableMetas, allDatabases, allFunctions); err != nil { - return err + backupMetaFile := path.Join(b.DefaultDataPath, "backup", backupName, "metadata.json") + if err := b.createBackupMetadata(ctx, backupMetaFile, backupName, diffFromRemote, backupVersion, "regular", diskMap, diskTypes, disks, backupDataSize, backupObjectDiskSize, backupMetadataSize, backupRBACSize, backupConfigSize, tableMetas, allDatabases, allFunctions); err != nil { + return fmt.Errorf("createBackupMetadata return error: %v", err) } - log.Info().Str("operation", "create").Str("duration", utils.HumanizeDuration(time.Since(startBackup))).Msg("done") + log.Info().Str("version", backupVersion).Str("duration", utils.HumanizeDuration(time.Since(startBackup))).Msg("done") return nil } -func (b *Backuper) createBackupEmbedded(ctx context.Context, backupName, tablePattern string, partitionsNameList map[metadata.TableTitle][]string, partitionsIdMap map[metadata.TableTitle]common.EmptyMap, schemaOnly, createRBAC, createConfigs bool, tables []clickhouse.Table, allDatabases []clickhouse.Database, allFunctions []clickhouse.Function, disks []clickhouse.Disk, diskMap, diskTypes map[string]string, startBackup time.Time, backupVersion string) error { +func (b *Backuper) createBackupEmbedded(ctx context.Context, backupName, baseBackup string, doBackupData, schemaOnly bool, backupVersion, tablePattern string, partitionsNameList map[metadata.TableTitle][]string, partitionsIdMap map[metadata.TableTitle]common.EmptyMap, tables []clickhouse.Table, allDatabases []clickhouse.Database, allFunctions []clickhouse.Function, disks []clickhouse.Disk, diskMap, diskTypes map[string]string, backupRBACSize, backupConfigSize uint64, startBackup time.Time, version int) error { // TODO: Implement sharded backup operations for embedded backups if doesShard(b.cfg.General.ShardedOperationMode) { return fmt.Errorf("cannot perform embedded backup: %w", errShardOperationUnsupported) } - if _, isBackupDiskExists := diskMap[b.cfg.ClickHouse.EmbeddedBackupDisk]; !isBackupDiskExists { - return fmt.Errorf("backup disk `%s` not exists in system.disks", b.cfg.ClickHouse.EmbeddedBackupDisk) + backupPath := path.Join(b.DefaultDataPath, "backup") + if b.cfg.ClickHouse.EmbeddedBackupDisk != "" { + backupPath = diskMap[b.cfg.ClickHouse.EmbeddedBackupDisk] } - if createRBAC || createConfigs { - return fmt.Errorf("`use_embedded_backup_restore: true` doesn't support --rbac, --configs parameters") + backupPath = path.Join(backupPath, backupName) + + backupMetadataSize := uint64(0) + backupDataSize := make([]clickhouse.BackupDataSize, 0) + if !schemaOnly && !doBackupData { + backupDataSize = append(backupDataSize, clickhouse.BackupDataSize{Size: 0}) } - l := 0 - for _, table := range tables { - if !table.Skip { - l += 1 + var tablesTitle []metadata.TableTitle + + if schemaOnly || doBackupData { + l := 0 + for _, table := range tables { + if !table.Skip { + l += 1 + } + } + if l == 0 { + return fmt.Errorf("`use_embedded_backup_restore: true` not found tables for backup, check your parameter --tables=%v", tablePattern) + } + tablesTitle = make([]metadata.TableTitle, l) + + if _, isBackupDiskExists := diskMap[b.cfg.ClickHouse.EmbeddedBackupDisk]; b.cfg.ClickHouse.EmbeddedBackupDisk != "" && !isBackupDiskExists { + return fmt.Errorf("backup disk `%s` not exists in system.disks", b.cfg.ClickHouse.EmbeddedBackupDisk) + } + if b.cfg.ClickHouse.EmbeddedBackupDisk == "" { + if err := config.ValidateObjectDiskConfig(b.cfg); err != nil { + return err + } + } + + backupSQL, tablesSizeSQL, err := b.generateEmbeddedBackupSQL(ctx, backupName, schemaOnly, tables, tablesTitle, partitionsNameList, l, baseBackup, version) + if err != nil { + return err + } + backupResult := make([]clickhouse.SystemBackups, 0) + if err := b.ch.SelectContext(ctx, &backupResult, backupSQL); err != nil { + return fmt.Errorf("backup error: %v", err) + } + if len(backupResult) != 1 || (backupResult[0].Status != "BACKUP_COMPLETE" && backupResult[0].Status != "BACKUP_CREATED") { + return fmt.Errorf("backup return wrong results: %+v", backupResult) + } + + if schemaOnly { + backupDataSize = append(backupDataSize, clickhouse.BackupDataSize{Size: 0}) + } else { + if backupResult[0].CompressedSize == 0 { + backupSizeSQL := "SELECT sum(bytes_on_disk) AS backup_data_size FROM system.parts WHERE active AND (" + for _, t := range tables { + if oneTableSizeSQL, exists := tablesSizeSQL[metadata.TableTitle{Database: t.Database, Table: t.Name}]; exists { + if strings.HasPrefix(oneTableSizeSQL, fmt.Sprintf("'%s::%s::", t.Database, t.Name)) { + backupSizeSQL += fmt.Sprintf(" concat(database,'::',table,'::',partition) IN (%s) OR ", oneTableSizeSQL) + } else { + backupSizeSQL += fmt.Sprintf(" concat(database,'::',table) IN (%s) OR ", tablesSizeSQL[metadata.TableTitle{Database: t.Database, Table: t.Name}]) + } + } + } + backupSizeSQL = backupSizeSQL[:len(backupSizeSQL)-4] + ")" + if err := b.ch.SelectContext(ctx, &backupDataSize, backupSizeSQL); err != nil { + return err + } + } else { + backupDataSize = append(backupDataSize, clickhouse.BackupDataSize{Size: backupResult[0].CompressedSize}) + } + } + + if doBackupData && b.cfg.ClickHouse.EmbeddedBackupDisk == "" { + var err error + if b.dst, err = storage.NewBackupDestination(ctx, b.cfg, b.ch, false, backupName); err != nil { + return err + } + if err = b.dst.Connect(ctx); err != nil { + return fmt.Errorf("createBackupEmbedded: can't connect to %s: %v", b.dst.Kind(), err) + } + defer func() { + if closeErr := b.dst.Close(ctx); closeErr != nil { + log.Warn().Msgf("createBackupEmbedded: can't close connection to %s: %v", b.dst.Kind(), closeErr) + } + }() + } + + for _, table := range tables { + select { + case <-ctx.Done(): + return ctx.Err() + default: + if table.Skip { + continue + } + var disksToPartsMap map[string][]metadata.Part + if doBackupData { + if b.cfg.ClickHouse.EmbeddedBackupDisk != "" { + log.Debug().Msgf("calculate parts list `%s`.`%s` from embedded backup disk `%s`", table.Database, table.Name, b.cfg.ClickHouse.EmbeddedBackupDisk) + disksToPartsMap, err = b.getPartsFromLocalEmbeddedBackupDisk(backupPath, table, partitionsIdMap[metadata.TableTitle{Database: table.Database, Table: table.Name}]) + } else { + log.Debug().Msgf("calculate parts list `%s`.`%s` from embedded backup remote destination", table.Database, table.Name) + disksToPartsMap, err = b.getPartsFromRemoteEmbeddedBackup(ctx, backupName, table, partitionsIdMap[metadata.TableTitle{Database: table.Database, Table: table.Name}]) + } + } + if err != nil { + return err + } + if schemaOnly || doBackupData { + metadataSize, err := b.createTableMetadata(path.Join(backupPath, "metadata"), metadata.TableMetadata{ + Table: table.Name, + Database: table.Database, + Query: table.CreateTableQuery, + TotalBytes: table.TotalBytes, + Size: map[string]int64{b.cfg.ClickHouse.EmbeddedBackupDisk: 0}, + Parts: disksToPartsMap, + MetadataOnly: schemaOnly, + }, disks) + if err != nil { + return err + } + backupMetadataSize += metadataSize + } + } } } - if l == 0 { - return fmt.Errorf("`use_embedded_backup_restore: true` doesn't allow empty backups, check your parameter --tables=%v", tablePattern) + backupMetaFile := path.Join(backupPath, "metadata.json") + if err := b.createBackupMetadata(ctx, backupMetaFile, backupName, baseBackup, backupVersion, "embedded", diskMap, diskTypes, disks, backupDataSize[0].Size, 0, backupMetadataSize, backupRBACSize, backupConfigSize, tablesTitle, allDatabases, allFunctions); err != nil { + return err } - tableMetas := make([]metadata.TableTitle, l) + + log.Info().Fields(map[string]interface{}{ + "operation": "create_embedded", + "duration": utils.HumanizeDuration(time.Since(startBackup)), + }).Msg("done") + + return nil +} + +func (b *Backuper) generateEmbeddedBackupSQL(ctx context.Context, backupName string, schemaOnly bool, tables []clickhouse.Table, tablesTitle []metadata.TableTitle, partitionsNameList map[metadata.TableTitle][]string, tablesListLen int, baseBackup string, version int) (string, map[metadata.TableTitle]string, error) { tablesSQL := "" - tableSizeSQL := "" + tableSizeSQL := map[metadata.TableTitle]string{} i := 0 - backupMetadataSize := uint64(0) - backupPath := path.Join(diskMap[b.cfg.ClickHouse.EmbeddedBackupDisk], backupName) for _, table := range tables { if table.Skip { continue } - tableMetas[i] = metadata.TableTitle{ + tablesTitle[i] = metadata.TableTitle{ Database: table.Database, Table: table.Name, } i += 1 tablesSQL += "TABLE `" + table.Database + "`.`" + table.Name + "`" - tableSizeSQL += "'" + table.Database + "." + table.Name + "'" + fullTableNameForTableSizeSQL := table.Database + "::" + table.Name + tableSizeSQLOneTable := "" if nameList, exists := partitionsNameList[metadata.TableTitle{Database: table.Database, Table: table.Name}]; exists && len(nameList) > 0 { - tablesSQL += fmt.Sprintf(" PARTITIONS '%s'", strings.Join(nameList, "','")) + partitionsSQL := "" + for _, partitionName := range nameList { + if partitionName != "*" { + tableSizeSQLOneTable += "'" + fullTableNameForTableSizeSQL + "::" + partitionName + "'," + if strings.HasPrefix(partitionName, "(") { + partitionsSQL += partitionName + "," + } else { + partitionsSQL += "'" + partitionName + "'," + } + } + } + tablesSQL += fmt.Sprintf(" PARTITIONS %s", partitionsSQL[:len(partitionsSQL)-1]) + if tableSizeSQLOneTable != "" { + tableSizeSQLOneTable = tableSizeSQLOneTable[:len(tableSizeSQLOneTable)-1] + } + } + if tableSizeSQLOneTable == "" { + tableSizeSQLOneTable = "'" + fullTableNameForTableSizeSQL + "'" } - if i < l { + tableSizeSQL[metadata.TableTitle{Database: table.Database, Table: table.Name}] = tableSizeSQLOneTable + if i < tablesListLen { tablesSQL += ", " - tableSizeSQL += ", " } } - backupSQL := fmt.Sprintf("BACKUP %s TO Disk(?,?)", tablesSQL) - if schemaOnly { - backupSQL += " SETTINGS structure_only=1, show_table_uuid_in_table_create_query_if_not_nil=1" - } - backupResult := make([]clickhouse.SystemBackups, 0) - if err := b.ch.SelectContext(ctx, &backupResult, backupSQL, b.cfg.ClickHouse.EmbeddedBackupDisk, backupName); err != nil { - return fmt.Errorf("backup error: %v", err) + backupSettings := b.getEmbeddedBackupDefaultSettings(version) + embeddedBackupLocation, err := b.getEmbeddedBackupLocation(ctx, backupName) + if err != nil { + return "", nil, err } - if len(backupResult) != 1 || (backupResult[0].Status != "BACKUP_COMPLETE" && backupResult[0].Status != "BACKUP_CREATED") { - return fmt.Errorf("backup return wrong results: %+v", backupResult) + backupSQL := fmt.Sprintf("BACKUP %s TO %s", tablesSQL, embeddedBackupLocation) + if schemaOnly { + backupSettings = append(backupSettings, "structure_only=1") } - backupDataSize := make([]struct { - Size uint64 `ch:"backup_data_size"` - }, 0) - if !schemaOnly { - if backupResult[0].CompressedSize == 0 { - chVersion, err := b.ch.GetVersion(ctx) - if err != nil { - return err - } - backupSizeSQL := fmt.Sprintf("SELECT sum(bytes_on_disk) AS backup_data_size FROM system.parts WHERE active AND concat(database,'.',table) IN (%s)", tableSizeSQL) - if chVersion >= 20005000 { - backupSizeSQL = fmt.Sprintf("SELECT sum(total_bytes) AS backup_data_size FROM system.tables WHERE concat(database,'.',name) IN (%s)", tableSizeSQL) - } - if err := b.ch.SelectContext(ctx, &backupDataSize, backupSizeSQL); err != nil { - return err - } - } else { - backupDataSize = append(backupDataSize, struct { - Size uint64 `ch:"backup_data_size"` - }{Size: backupResult[0].CompressedSize}) + // incremental native backup https://github.com/Altinity/clickhouse-backup/issues/735 + if baseBackup != "" { + baseBackup, err = b.getEmbeddedBackupLocation(ctx, baseBackup) + if err != nil { + return "", nil, err } - } else { - backupDataSize = append(backupDataSize, struct { - Size uint64 `ch:"backup_data_size"` - }{Size: 0}) + backupSettings = append(backupSettings, "base_backup="+baseBackup) } + if len(backupSettings) > 0 { + backupSQL += " SETTINGS " + strings.Join(backupSettings, ", ") + } + return backupSQL, tableSizeSQL, nil +} - log.Debug().Msg("calculate parts list from embedded backup disk") - for _, table := range tables { - select { - case <-ctx.Done(): - return ctx.Err() - default: - if table.Skip { - continue - } - disksToPartsMap, err := b.getPartsFromBackupDisk(backupPath, table, partitionsIdMap[metadata.TableTitle{Database: table.Database, Table: table.Name}]) - if err != nil { - if removeBackupErr := b.RemoveBackupLocal(ctx, backupName, disks); removeBackupErr != nil { - log.Error().Msg(removeBackupErr.Error()) - } - return err - } - metadataSize, err := b.createTableMetadata(path.Join(backupPath, "metadata"), metadata.TableMetadata{ - Table: table.Name, - Database: table.Database, - Query: table.CreateTableQuery, - TotalBytes: table.TotalBytes, - Size: map[string]int64{b.cfg.ClickHouse.EmbeddedBackupDisk: 0}, - Parts: disksToPartsMap, - MetadataOnly: schemaOnly, - }, disks) - if err != nil { - if removeBackupErr := b.RemoveBackupLocal(ctx, backupName, disks); removeBackupErr != nil { - log.Error().Msg(removeBackupErr.Error()) - } - return err - } - backupMetadataSize += metadataSize - } +func (b *Backuper) getPartsFromRemoteEmbeddedBackup(ctx context.Context, backupName string, table clickhouse.Table, partitionsIdsMap common.EmptyMap) (map[string][]metadata.Part, error) { + dirListStr := make([]string, 0) + remoteEmbeddedBackupPath := "" + if b.cfg.General.RemoteStorage == "s3" { + remoteEmbeddedBackupPath = b.cfg.S3.ObjectDiskPath + } else if b.cfg.General.RemoteStorage == "gcs" { + remoteEmbeddedBackupPath = b.cfg.GCS.ObjectDiskPath + } else if b.cfg.General.RemoteStorage == "azblob" { + remoteEmbeddedBackupPath = b.cfg.AzureBlob.ObjectDiskPath + } else { + return nil, fmt.Errorf("getPartsFromRemoteEmbeddedBackup: unsupported remote_storage: %s", b.cfg.General.RemoteStorage) } - backupMetaFile := path.Join(diskMap[b.cfg.ClickHouse.EmbeddedBackupDisk], backupName, "metadata.json") - if err := b.createBackupMetadata(ctx, backupMetaFile, backupName, backupVersion, "embedded", diskMap, diskTypes, disks, backupDataSize[0].Size, backupMetadataSize, 0, 0, tableMetas, allDatabases, allFunctions); err != nil { - return err + remoteEmbeddedBackupPath = path.Join(remoteEmbeddedBackupPath, backupName, "data", common.TablePathEncode(table.Database), common.TablePathEncode(table.Name)) + if walkErr := b.dst.WalkAbsolute(ctx, remoteEmbeddedBackupPath, false, func(ctx context.Context, fInfo storage.RemoteFile) error { + dirListStr = append(dirListStr, fInfo.Name()) + return nil + }); walkErr != nil { + return nil, walkErr } - - log.Info().Fields(map[string]interface{}{ - "operation": "create_embedded", - "duration": utils.HumanizeDuration(time.Since(startBackup)), - }).Msg("done") - - return nil + log.Debug().Msgf("getPartsFromRemoteEmbeddedBackup from %s found %d parts", remoteEmbeddedBackupPath, len(dirListStr)) + return b.fillEmbeddedPartsFromDirList(partitionsIdsMap, dirListStr, "default") } -func (b *Backuper) getPartsFromBackupDisk(backupPath string, table clickhouse.Table, partitionsIdsMap common.EmptyMap) (map[string][]metadata.Part, error) { - parts := map[string][]metadata.Part{} +func (b *Backuper) getPartsFromLocalEmbeddedBackupDisk(backupPath string, table clickhouse.Table, partitionsIdsMap common.EmptyMap) (map[string][]metadata.Part, error) { dirList, err := os.ReadDir(path.Join(backupPath, "data", common.TablePathEncode(table.Database), common.TablePathEncode(table.Name))) if err != nil { if os.IsNotExist(err) { - return parts, nil + return map[string][]metadata.Part{}, nil } return nil, err } + dirListStr := make([]string, len(dirList)) + for i, d := range dirList { + dirListStr[i] = d.Name() + } + return b.fillEmbeddedPartsFromDirList(partitionsIdsMap, dirListStr, b.cfg.ClickHouse.EmbeddedBackupDisk) +} + +func (b *Backuper) fillEmbeddedPartsFromDirList(partitionsIdsMap common.EmptyMap, dirList []string, diskName string) (map[string][]metadata.Part, error) { + parts := map[string][]metadata.Part{} if len(partitionsIdsMap) == 0 { - parts[b.cfg.ClickHouse.EmbeddedBackupDisk] = make([]metadata.Part, len(dirList)) - for i, d := range dirList { - parts[b.cfg.ClickHouse.EmbeddedBackupDisk][i] = metadata.Part{ - Name: d.Name(), + parts[diskName] = make([]metadata.Part, len(dirList)) + for i, dirName := range dirList { + parts[diskName][i] = metadata.Part{ + Name: dirName, } } - } else { - parts[b.cfg.ClickHouse.EmbeddedBackupDisk] = make([]metadata.Part, 0) - for _, d := range dirList { - found := false - for prefix := range partitionsIdsMap { - if strings.HasPrefix(d.Name(), prefix+"_") { + return parts, nil + } + + parts[diskName] = make([]metadata.Part, 0) + for _, dirName := range dirList { + found := false + for prefix := range partitionsIdsMap { + if strings.HasPrefix(dirName, prefix+"_") { + found = true + break + } + if strings.Contains(prefix, "*") { + if matched, err := filepath.Match(dirName, prefix); err == nil && matched { found = true break } } - if found { - parts[b.cfg.ClickHouse.EmbeddedBackupDisk] = append(parts[b.cfg.ClickHouse.EmbeddedBackupDisk], metadata.Part{ - Name: d.Name(), - }) - } + } + if found { + parts[diskName] = append(parts[diskName], metadata.Part{ + Name: dirName, + }) } } return parts, nil @@ -438,36 +638,46 @@ func (b *Backuper) createBackupRBAC(ctx context.Context, backupPath string, disk default: rbacDataSize := uint64(0) rbacBackup := path.Join(backupPath, "access") + replicatedRBACDataSize, err := b.createBackupRBACReplicated(ctx, rbacBackup) + if err != nil { + return 0, err + } accessPath, err := b.ch.GetAccessManagementPath(ctx, disks) if err != nil { return 0, err } accessPathInfo, err := os.Stat(accessPath) if err != nil && !os.IsNotExist(err) { - return 0, err + return rbacDataSize + replicatedRBACDataSize, err } if err == nil && !accessPathInfo.IsDir() { - return 0, fmt.Errorf("%s is not directory", accessPath) + return rbacDataSize + replicatedRBACDataSize, fmt.Errorf("%s is not directory", accessPath) + } + if os.IsNotExist(err) { + return rbacDataSize + replicatedRBACDataSize, nil } - if err == nil { + rbacSQLFiles, err := filepath.Glob(path.Join(accessPath, "*.sql")) + if err != nil { + return rbacDataSize + replicatedRBACDataSize, err + } + if len(rbacSQLFiles) != 0 { log.Debug().Msgf("copy %s -> %s", accessPath, rbacBackup) copyErr := recursiveCopy.Copy(accessPath, rbacBackup, recursiveCopy.Options{ - Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { - rbacDataSize += uint64(srcinfo.Size()) - return false, nil + OnDirExists: func(src, dst string) recursiveCopy.DirExistsAction { + return recursiveCopy.Replace + }, + Skip: func(srcinfo os.FileInfo, src, dst string) (bool, error) { + if strings.HasSuffix(src, ".sql") { + rbacDataSize += uint64(srcinfo.Size()) + return false, nil + } else { + return true, nil + } }, }) if copyErr != nil { return 0, copyErr } - } else { - if err = os.MkdirAll(rbacBackup, 0755); err != nil { - return 0, err - } - } - replicatedRBACDataSize, err := b.createBackupRBACReplicated(ctx, rbacBackup) - if err != nil { - return 0, err } return rbacDataSize + replicatedRBACDataSize, nil } @@ -489,6 +699,17 @@ func (b *Backuper) createBackupRBACReplicated(ctx context.Context, rbacBackup st if err != nil { return 0, err } + rbacUUIDObjectsCount, err := k.ChildCount(replicatedAccessPath, "uuid") + if err != nil { + return 0, err + } + if rbacUUIDObjectsCount == 0 { + log.Warn().Str("logger", "createBackupRBACReplicated").Msgf("%s/%s have no childs, skip Dump", replicatedAccessPath, "uuid") + continue + } + if err = os.MkdirAll(rbacBackup, 0755); err != nil { + return 0, err + } dumpFile := path.Join(rbacBackup, userDirectory.Name+".jsonl") log.Info().Str("logger", "createBackupRBACReplicated").Msgf("keeper.Dump %s -> %s", replicatedAccessPath, dumpFile) dumpRBACSize, dumpErr := k.Dump(replicatedAccessPath, dumpFile) @@ -501,43 +722,40 @@ func (b *Backuper) createBackupRBACReplicated(ctx context.Context, rbacBackup st return rbacDataSize, nil } -func (b *Backuper) AddTableToBackup(ctx context.Context, backupName, shadowBackupUUID string, diskList []clickhouse.Disk, table *clickhouse.Table, partitionsIdsMap common.EmptyMap) (map[string][]metadata.Part, map[string]int64, error) { +func (b *Backuper) AddTableToLocalBackup(ctx context.Context, backupName string, tablesDiffFromRemote map[metadata.TableTitle]metadata.TableMetadata, shadowBackupUUID string, diskList []clickhouse.Disk, table *clickhouse.Table, partitionsIdsMap common.EmptyMap, version int) (map[string][]metadata.Part, map[string]int64, map[string]int64, error) { logger := log.With().Fields(map[string]interface{}{ "backup": backupName, "operation": "create", "table": fmt.Sprintf("%s.%s", table.Database, table.Name), }).Logger() if backupName == "" { - return nil, nil, fmt.Errorf("backupName is not defined") + return nil, nil, nil, fmt.Errorf("backupName is not defined") } if !strings.HasSuffix(table.Engine, "MergeTree") && table.Engine != "MaterializedMySQL" && table.Engine != "MaterializedPostgreSQL" { if table.Engine != "MaterializedView" { logger.Warn().Str("engine", table.Engine).Msg("supports only schema backup") } - return nil, nil, nil + return nil, nil, nil, nil } if b.cfg.ClickHouse.CheckPartsColumns { if err := b.ch.CheckSystemPartsColumns(ctx, table); err != nil { - return nil, nil, err + return nil, nil, nil, err } } // backup data if err := b.ch.FreezeTable(ctx, table, shadowBackupUUID); err != nil { - return nil, nil, err - } - logger.Debug().Msg("frozen") - version, err := b.ch.GetVersion(ctx) - if err != nil { - return nil, nil, err + return nil, nil, nil, err } + log.Debug().Msg("frozen") realSize := map[string]int64{} + objectDiskSize := map[string]int64{} disksToPartsMap := map[string][]metadata.Part{} for _, disk := range diskList { select { case <-ctx.Done(): - return nil, nil, ctx.Err() + return nil, nil, nil, ctx.Err() default: shadowPath := path.Join(disk.Path, "shadow", shadowBackupUUID) if _, err := os.Stat(shadowPath); err != nil && os.IsNotExist(err) { @@ -547,40 +765,31 @@ func (b *Backuper) AddTableToBackup(ctx context.Context, backupName, shadowBacku encodedTablePath := path.Join(common.TablePathEncode(table.Database), common.TablePathEncode(table.Name)) backupShadowPath := path.Join(backupPath, "shadow", encodedTablePath, disk.Name) if err := filesystemhelper.MkdirAll(backupShadowPath, b.ch, diskList); err != nil && !os.IsExist(err) { - return nil, nil, err + return nil, nil, nil, err } // If partitionsIdsMap is not empty, only parts in this partition will back up. - parts, size, err := filesystemhelper.MoveShadow(shadowPath, backupShadowPath, partitionsIdsMap) + parts, size, err := filesystemhelper.MoveShadowToBackup(shadowPath, backupShadowPath, partitionsIdsMap, tablesDiffFromRemote[metadata.TableTitle{Database: table.Database, Table: table.Name}], disk, version) if err != nil { - return nil, nil, err + return nil, nil, nil, err } realSize[disk.Name] = size + disksToPartsMap[disk.Name] = parts logger.Debug().Str("disk", disk.Name).Msg("shadow moved") - if disk.Type == "s3" || disk.Type == "azure_blob_storage" && len(parts) > 0 { - if err = config.ValidateObjectDiskConfig(b.cfg); err != nil { - return nil, nil, err - } + if len(parts) > 0 && (b.isDiskTypeObject(disk.Type) || b.isDiskTypeEncryptedObject(disk, diskList)) { start := time.Now() - if b.dst == nil { - b.dst, err = storage.NewBackupDestination(ctx, b.cfg, b.ch, false, backupName) - if err != nil { - return nil, nil, err - } - if err := b.dst.Connect(ctx); err != nil { - return nil, nil, fmt.Errorf("can't connect to %s: %v", b.dst.Kind(), err) - } + if size, err = b.uploadObjectDiskParts(ctx, backupName, tablesDiffFromRemote[metadata.TableTitle{Database: table.Database, Table: table.Name}], backupShadowPath, disk); err != nil { + return nil, nil, nil, err } - if size, err = b.uploadObjectDiskParts(ctx, backupName, backupShadowPath, disk); err != nil { - return disksToPartsMap, realSize, err + objectDiskSize[disk.Name] = size + if size > 0 { + log.Info().Str("disk", disk.Name).Str("duration", utils.HumanizeDuration(time.Since(start))).Str("size", utils.FormatBytes(uint64(size))).Msg("upload object_disk finish") } - realSize[disk.Name] += size - logger.Info().Str("disk", disk.Name).Str("duration", utils.HumanizeDuration(time.Since(start))).Msg("object_disk data uploaded") } // Clean all the files under the shadowPath, cause UNFREEZE unavailable if version < 21004000 { if err := os.RemoveAll(shadowPath); err != nil { - return disksToPartsMap, realSize, err + return nil, nil, nil, err } } } @@ -590,75 +799,99 @@ func (b *Backuper) AddTableToBackup(ctx context.Context, backupName, shadowBacku if err := b.ch.QueryContext(ctx, fmt.Sprintf("ALTER TABLE `%s`.`%s` UNFREEZE WITH NAME '%s'", table.Database, table.Name, shadowBackupUUID)); err != nil { if (strings.Contains(err.Error(), "code: 60") || strings.Contains(err.Error(), "code: 81") || strings.Contains(err.Error(), "code: 218")) && b.cfg.ClickHouse.IgnoreNotExistsErrorDuringFreeze { logger.Warn().Msgf("can't unfreeze table: %v", err) - } else { - return disksToPartsMap, realSize, err } - - } - } - if b.dst != nil { - if err := b.dst.Close(ctx); err != nil { - logger.Warn().Msgf("uploadObjectDiskParts: can't close BackupDestination error: %v", err) } } - logger.Debug().Msg("done") - return disksToPartsMap, realSize, nil + log.Debug().Msg("done") + return disksToPartsMap, realSize, objectDiskSize, nil } -func (b *Backuper) uploadObjectDiskParts(ctx context.Context, backupName, backupShadowPath string, disk clickhouse.Disk) (int64, error) { +func (b *Backuper) uploadObjectDiskParts(ctx context.Context, backupName string, tableDiffFromRemote metadata.TableMetadata, backupShadowPath string, disk clickhouse.Disk) (int64, error) { var size int64 var err error if err = object_disk.InitCredentialsAndConnections(ctx, b.ch, b.cfg, disk.Name); err != nil { return 0, err } - - if err := filepath.Walk(backupShadowPath, func(fPath string, fInfo os.FileInfo, err error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + uploadObjectDiskPartsWorkingGroup, ctx := errgroup.WithContext(ctx) + uploadObjectDiskPartsWorkingGroup.SetLimit(int(b.cfg.General.ObjectDiskServerSideCopyConcurrency)) + srcDiskConnection, exists := object_disk.DisksConnections.Load(disk.Name) + if !exists { + return 0, fmt.Errorf("uploadObjectDiskParts: %s not present in object_disk.DisksConnections", disk.Name) + } + srcBucket := srcDiskConnection.GetRemoteBucket() + walkErr := filepath.Walk(backupShadowPath, func(fPath string, fInfo os.FileInfo, err error) error { if err != nil { return err } if fInfo.IsDir() { return nil } - objPartFileMeta, err := object_disk.ReadMetadataFromFile(fPath) - if err != nil { - return err + // fix https://github.com/Altinity/clickhouse-backup/issues/826 + if strings.Contains(fInfo.Name(), "frozen_metadata") { + return nil } var realSize, objSize int64 - // @TODO think about parallelization here after test pass - for _, storageObject := range objPartFileMeta.StorageObjects { - srcDiskConnection, exists := object_disk.DisksConnections[disk.Name] - if !exists { - return fmt.Errorf("uploadObjectDiskParts: %s not present in object_disk.DisksConnections", disk.Name) - } - if objSize, err = b.dst.CopyObject( - ctx, - srcDiskConnection.GetRemoteBucket(), - path.Join(srcDiskConnection.GetRemotePath(), storageObject.ObjectRelativePath), - path.Join(backupName, disk.Name, storageObject.ObjectRelativePath), - ); err != nil { - return err + // upload only not required parts, https://github.com/Altinity/clickhouse-backup/issues/865 + if tableDiffFromRemote.Database != "" && tableDiffFromRemote.Table != "" && len(tableDiffFromRemote.Parts[disk.Name]) > 0 { + partPaths := strings.SplitN(strings.TrimPrefix(fPath, backupShadowPath), "/", 2) + for _, part := range tableDiffFromRemote.Parts[disk.Name] { + if part.Name == partPaths[0] { + log.Debug().Msgf("%s exists in diff-from-remote backup", part.Name) + return nil + } } - realSize += objSize - } - if realSize > objPartFileMeta.TotalSize { - size += realSize - } else { - size += objPartFileMeta.TotalSize } + uploadObjectDiskPartsWorkingGroup.Go(func() error { + objPartFileMeta, readMetadataErr := object_disk.ReadMetadataFromFile(fPath) + if readMetadataErr != nil { + return readMetadataErr + } + for _, storageObject := range objPartFileMeta.StorageObjects { + //b.log.WithField("object_file", fPath).WithField("size", storageObject.ObjectSize).WithField("object_key", storageObject.ObjectRelativePath).Debug("prepare CopyObject") + if storageObject.ObjectSize == 0 { + continue + } + //b.log.WithField("object_file", fPath).Debug("start copy") + if objSize, err = b.dst.CopyObject( + ctx, + storageObject.ObjectSize, + srcBucket, + path.Join(srcDiskConnection.GetRemotePath(), storageObject.ObjectRelativePath), + path.Join(backupName, disk.Name, storageObject.ObjectRelativePath), + ); err != nil { + return err + } + realSize += objSize + } + if realSize > objPartFileMeta.TotalSize { + atomic.AddInt64(&size, realSize) + } else { + atomic.AddInt64(&size, objPartFileMeta.TotalSize) + } + return nil + }) return nil - }); err != nil { + }) + if walkErr != nil { return 0, err } + + if wgWaitErr := uploadObjectDiskPartsWorkingGroup.Wait(); wgWaitErr != nil { + return 0, fmt.Errorf("one of uploadObjectDiskParts go-routine return error: %v", wgWaitErr) + } return size, nil } -func (b *Backuper) createBackupMetadata(ctx context.Context, backupMetaFile, backupName, version, tags string, diskMap, diskTypes map[string]string, disks []clickhouse.Disk, backupDataSize, backupMetadataSize, backupRBACSize, backupConfigSize uint64, tableMetas []metadata.TableTitle, allDatabases []clickhouse.Database, allFunctions []clickhouse.Function) error { +func (b *Backuper) createBackupMetadata(ctx context.Context, backupMetaFile, backupName, requiredBackup, version, tags string, diskMap, diskTypes map[string]string, disks []clickhouse.Disk, backupDataSize, backupObjectDiskSize, backupMetadataSize, backupRBACSize, backupConfigSize uint64, tableMetas []metadata.TableTitle, allDatabases []clickhouse.Database, allFunctions []clickhouse.Function) error { select { case <-ctx.Done(): return ctx.Err() default: backupMetadata := metadata.BackupMetadata{ BackupName: backupName, + RequiredBackup: requiredBackup, Disks: diskMap, DiskTypes: diskTypes, ClickhouseBackupVersion: version, @@ -666,6 +899,7 @@ func (b *Backuper) createBackupMetadata(ctx context.Context, backupMetaFile, bac Tags: tags, ClickHouseVersion: b.ch.GetVersionDescribe(ctx), DataSize: backupDataSize, + ObjectDiskSize: backupObjectDiskSize, MetadataSize: backupMetadataSize, RBACSize: backupRBACSize, ConfigSize: backupConfigSize, @@ -681,16 +915,15 @@ func (b *Backuper) createBackupMetadata(ctx context.Context, backupMetaFile, bac } content, err := json.MarshalIndent(&backupMetadata, "", "\t") if err != nil { - _ = b.RemoveBackupLocal(ctx, backupName, disks) return fmt.Errorf("can't marshal backup metafile json: %v", err) } if err := os.WriteFile(backupMetaFile, content, 0640); err != nil { - _ = b.RemoveBackupLocal(ctx, backupName, disks) return err } if err := filesystemhelper.Chown(backupMetaFile, b.ch, disks, false); err != nil { log.Warn().Msgf("can't chown %s: %v", backupMetaFile, err) } + log.Debug().Msgf("%s created", backupMetaFile) return nil } } @@ -714,5 +947,6 @@ func (b *Backuper) createTableMetadata(metadataPath string, table metadata.Table if err := filesystemhelper.Chown(metadataFile, b.ch, disks, false); err != nil { return 0, err } + log.Debug().Msgf("%s created", metadataFile) return uint64(len(metadataBody)), nil } diff --git a/pkg/backup/create_remote.go b/pkg/backup/create_remote.go index 863a05e4..646a5811 100644 --- a/pkg/backup/create_remote.go +++ b/pkg/backup/create_remote.go @@ -2,11 +2,10 @@ package backup import ( "context" - "fmt" - "github.com/Altinity/clickhouse-backup/pkg/status" + "github.com/Altinity/clickhouse-backup/v2/pkg/status" ) -func (b *Backuper) CreateToRemote(backupName, diffFrom, diffFromRemote, tablePattern string, partitions []string, schemaOnly, backupRBAC, rbacOnly, backupConfigs, configsOnly, skipCheckPartsColumns, resume bool, version string, commandId int) error { +func (b *Backuper) CreateToRemote(backupName string, deleteSource bool, diffFrom, diffFromRemote, tablePattern string, partitions []string, schemaOnly, backupRBAC, rbacOnly, backupConfigs, configsOnly, skipCheckPartsColumns, resume bool, version string, commandId int) error { ctx, cancel, err := status.Current.GetContextWithCancel(commandId) if err != nil { return err @@ -16,15 +15,12 @@ func (b *Backuper) CreateToRemote(backupName, diffFrom, diffFromRemote, tablePat if backupName == "" { backupName = NewBackupName() } - if err := b.CreateBackup(backupName, tablePattern, partitions, schemaOnly, backupRBAC, rbacOnly, backupConfigs, configsOnly, skipCheckPartsColumns, version, commandId); err != nil { + if err := b.CreateBackup(backupName, diffFromRemote, tablePattern, partitions, schemaOnly, backupRBAC, rbacOnly, backupConfigs, configsOnly, skipCheckPartsColumns, version, commandId); err != nil { return err } - if err := b.Upload(backupName, diffFrom, diffFromRemote, tablePattern, partitions, schemaOnly, resume, commandId); err != nil { + if err := b.Upload(backupName, deleteSource, diffFrom, diffFromRemote, tablePattern, partitions, schemaOnly, resume, version, commandId); err != nil { return err } - if err := b.RemoveOldBackupsLocal(ctx, false, nil); err != nil { - return fmt.Errorf("can't remove old local backups: %v", err) - } return nil } diff --git a/pkg/backup/delete.go b/pkg/backup/delete.go index ee8bced9..7f08f23d 100644 --- a/pkg/backup/delete.go +++ b/pkg/backup/delete.go @@ -3,21 +3,19 @@ package backup import ( "context" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/storage/object_disk" - apexLog "github.com/apex/log" "io/fs" "os" "path" "path/filepath" - "regexp" "strings" "time" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/custom" - "github.com/Altinity/clickhouse-backup/pkg/status" - "github.com/Altinity/clickhouse-backup/pkg/storage" - "github.com/Altinity/clickhouse-backup/pkg/utils" + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/custom" + "github.com/Altinity/clickhouse-backup/v2/pkg/status" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage/object_disk" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" "github.com/pkg/errors" "github.com/rs/zerolog/log" @@ -99,10 +97,10 @@ func (b *Backuper) RemoveOldBackupsLocal(ctx context.Context, keepLastBackup boo if err != nil { return err } - backupsToDelete := GetBackupsToDelete(backupList, keep) + backupsToDelete := GetBackupsToDeleteLocal(backupList, keep) for _, backup := range backupsToDelete { - if err := b.RemoveBackupLocal(ctx, backup.BackupName, disks); err != nil { - return err + if deleteErr := b.RemoveBackupLocal(ctx, backup.BackupName, disks); deleteErr != nil { + return deleteErr } } return nil @@ -126,49 +124,37 @@ func (b *Backuper) RemoveBackupLocal(ctx context.Context, backupName string, dis if err != nil { return err } - - if b.hasObjectDisks(backupList, backupName, disks) { - bd, err := storage.NewBackupDestination(ctx, b.cfg, b.ch, false, backupName) - if err != nil { - return err - } - err = bd.Connect(ctx) - if err != nil { - return fmt.Errorf("can't connect to remote storage: %v", err) - } - defer func() { - if err := bd.Close(ctx); err != nil { - log.Warn().Msgf("can't close BackupDestination error: %v", err) - } - }() - b.dst = bd - } + hasObjectDisks := b.hasObjectDisksLocal(backupList, backupName, disks) for _, backup := range backupList { if backup.BackupName == backupName { - var skip bool - skip, err = b.skipIfTheSameRemoteBackupPresent(ctx, backup.BackupName, backup.Tags) - if err != nil { - return err - } - if !skip && strings.Contains(backup.Tags, "embedded") { - if err = b.cleanLocalEmbedded(ctx, backup, disks); err != nil { - log.Warn().Msgf("b.cleanRemoteEmbedded return error: %v", err) + b.isEmbedded = strings.Contains(backup.Tags, "embedded") + if hasObjectDisks || (b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk == "") { + bd, err := storage.NewBackupDestination(ctx, b.cfg, b.ch, false, backupName) + if err != nil { return err } + err = bd.Connect(ctx) + if err != nil { + return fmt.Errorf("can't connect to remote storage: %v", err) + } + defer func() { + if err := bd.Close(ctx); err != nil { + log.Warn().Msgf("can't close BackupDestination error: %v", err) + } + }() + b.dst = bd + } + err = b.cleanEmbeddedAndObjectDiskLocalIfSameRemoteNotPresent(ctx, backupName, disks, backup, hasObjectDisks) + if err != nil { + return err } - for _, disk := range disks { backupPath := path.Join(disk.Path, "backup", backupName) if disk.IsBackup { backupPath = path.Join(disk.Path, backupName) } - if !skip && !disk.IsBackup && (disk.Type == "s3" || disk.Type == "azure_blob_storage") && !strings.Contains(backup.Tags, "embedded") { - if err = b.cleanLocalBackupObjectDisk(ctx, backupName, backupPath, disk.Name); err != nil { - return err - } - } - log.Debug().Msgf("remove '%s'", backupPath) + log.Info().Msgf("remove '%s'", backupPath) if err = os.RemoveAll(backupPath); err != nil { return err } @@ -184,13 +170,38 @@ func (b *Backuper) RemoveBackupLocal(ctx context.Context, backupName string, dis return fmt.Errorf("'%s' is not found on local storage", backupName) } -func (b *Backuper) hasObjectDisks(backupList []LocalBackup, backupName string, disks []clickhouse.Disk) bool { +func (b *Backuper) cleanEmbeddedAndObjectDiskLocalIfSameRemoteNotPresent(ctx context.Context, backupName string, disks []clickhouse.Disk, backup LocalBackup, hasObjectDisks bool) error { + skip, err := b.skipIfTheSameRemoteBackupPresent(ctx, backup.BackupName, backup.Tags) + log.Debug().Msgf("b.skipIfTheSameRemoteBackupPresent return skip=%v", skip) + if err != nil { + return err + } + if !skip && (hasObjectDisks || (b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk == "")) { + startTime := time.Now() + if deletedKeys, deleteErr := b.cleanBackupObjectDisks(ctx, backupName); deleteErr != nil { + log.Warn().Msgf("b.cleanBackupObjectDisks return error: %v", deleteErr) + return err + } else { + log.Info().Str("backup", backupName).Str("duration", utils.HumanizeDuration(time.Since(startTime))).Msgf("cleanBackupObjectDisks deleted %d keys", deletedKeys) + } + } + if !skip && (b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk != "") { + if err = b.cleanLocalEmbedded(ctx, backup, disks); err != nil { + log.Warn().Msgf("b.cleanLocalEmbedded return error: %v", err) + return err + } + } + return nil +} + +func (b *Backuper) hasObjectDisksLocal(backupList []LocalBackup, backupName string, disks []clickhouse.Disk) bool { for _, backup := range backupList { - if backup.BackupName == backupName && !strings.Contains(backup.Tags, "embedded") { + if backup.BackupName == backupName && !b.isEmbedded { for _, disk := range disks { - if !disk.IsBackup && (disk.Type == "s3" || disk.Type == "azure_blob_storage") { + if !disk.IsBackup && (b.isDiskTypeObject(disk.Type) || b.isDiskTypeEncryptedObject(disk, disks)) { backupExists, err := os.ReadDir(path.Join(disk.Path, "backup", backup.BackupName)) if err == nil && len(backupExists) > 0 { + log.Debug().Msgf("hasObjectDisksLocal: found object disk %s", disk.Name) return true } } @@ -200,37 +211,9 @@ func (b *Backuper) hasObjectDisks(backupList []LocalBackup, backupName string, d return false } -func (b *Backuper) cleanLocalBackupObjectDisk(ctx context.Context, backupName string, backupPath, diskName string) error { - _, err := os.Stat(backupPath) - if os.IsNotExist(err) { - return nil - } else if err != nil { - return err - } - err = filepath.Walk(backupPath, func(fPath string, fInfo os.FileInfo, err error) error { - if err != nil { - return err - } - if fInfo.IsDir() { - return nil - } - objMeta, err := object_disk.ReadMetadataFromFile(fPath) - if err != nil { - return err - } - for _, storageObject := range objMeta.StorageObjects { - if err = b.dst.DeleteFileFromObjectDiskBackup(ctx, path.Join(backupName, diskName, storageObject.ObjectRelativePath)); err != nil { - return err - } - } - return nil - }) - return err -} - func (b *Backuper) cleanLocalEmbedded(ctx context.Context, backup LocalBackup, disks []clickhouse.Disk) error { for _, disk := range disks { - if disk.Name == b.cfg.ClickHouse.EmbeddedBackupDisk { + if disk.Name == b.cfg.ClickHouse.EmbeddedBackupDisk && disk.Type != "local" { if err := object_disk.InitCredentialsAndConnections(ctx, b.ch, b.cfg, disk.Name); err != nil { return err } @@ -239,8 +222,8 @@ func (b *Backuper) cleanLocalEmbedded(ctx context.Context, backup LocalBackup, d if err != nil { return err } - if !info.IsDir() && !strings.HasSuffix(filePath, ".json") { - apexLog.Debugf("object_disk.ReadMetadataFromFile(%s)", filePath) + if !info.IsDir() && !strings.HasSuffix(filePath, ".json") && !strings.HasPrefix(filePath, path.Join(backupPath, "access")) { + log.Debug().Msgf("object_disk.ReadMetadataFromFile(%s)", filePath) meta, err := object_disk.ReadMetadataFromFile(filePath) if err != nil { return err @@ -316,21 +299,12 @@ func (b *Backuper) RemoveBackupRemote(ctx context.Context, backupName string) er } for _, backup := range backupList { if backup.BackupName == backupName { - if skip, err := b.skipIfSameLocalBackupPresent(ctx, backup.BackupName, backup.Tags); err != nil { + err = b.cleanEmbeddedAndObjectDiskRemoteIfSameLocalNotPresent(ctx, backup) + if err != nil { return err - } else if !skip { - if strings.Contains(backup.Tags, "embedded") { - if err = b.cleanRemoteEmbedded(ctx, backup, bd); err != nil { - log.Warn().Msgf("b.cleanRemoteEmbedded return error: %v", err) - return err - } - } else if err = b.cleanRemoteBackupObjectDisks(ctx, backup); err != nil { - log.Warn().Msgf("b.cleanRemoteBackupObjectDisks return error: %v", err) - return err - } } - if err = bd.RemoveBackup(ctx, backup); err != nil { + if err = bd.RemoveBackupRemote(ctx, backup); err != nil { log.Warn().Msgf("bd.RemoveBackup return error: %v", err) return err } @@ -346,85 +320,53 @@ func (b *Backuper) RemoveBackupRemote(ctx context.Context, backupName string) er return fmt.Errorf("'%s' is not found on remote storage", backupName) } -func (b *Backuper) cleanRemoteBackupObjectDisks(ctx context.Context, backup storage.Backup) error { - if b.dst.Kind() != "azblob" && b.dst.Kind() != "s3" && b.dst.Kind() != "gcs" { - return nil - } - if !backup.Legacy && len(backup.Disks) > 0 && backup.DiskTypes != nil && len(backup.DiskTypes) < len(backup.Disks) { - return fmt.Errorf("RemoveRemoteBackupObjectDisks: invalid backup.DiskTypes=%#v, not correlated with backup.Disks=%#v", backup.DiskTypes, backup.Disks) +func (b *Backuper) cleanEmbeddedAndObjectDiskRemoteIfSameLocalNotPresent(ctx context.Context, backup storage.Backup) error { + var skip bool + var err error + if skip, err = b.skipIfSameLocalBackupPresent(ctx, backup.BackupName, backup.Tags); err != nil { + return err } - return b.dst.Walk(ctx, backup.BackupName+"/", true, func(ctx context.Context, f storage.RemoteFile) error { - fName := path.Join(backup.BackupName, f.Name()) - if !strings.HasPrefix(fName, path.Join(backup.BackupName, "/shadow/")) { + if !skip { + if b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk != "" { + if err = b.cleanRemoteEmbedded(ctx, backup); err != nil { + log.Warn().Msgf("b.cleanRemoteEmbedded return error: %v", err) + return err + } return nil } - for diskName, diskType := range backup.DiskTypes { - if diskType == "s3" || diskType == "azure_blob_storage" { - compressedRE := regexp.MustCompile(`/shadow/([^/]+/[^/]+)/` + diskName + `_[^/]+$`) - if matches := compressedRE.FindStringSubmatch(fName); len(matches) > 0 { - // compressed remote object disk part - localPath := path.Join(backup.Disks[diskName], "backup", backup.BackupName, "shadow", matches[1], diskName) - if err := b.dst.DownloadCompressedStream(ctx, fName, localPath); err != nil { - return err - } - if err := filepath.Walk(localPath, func(fPath string, fInfo fs.FileInfo, err error) error { - if err != nil { - return err - } - if fInfo.IsDir() { - return nil - } - objMeta, err := object_disk.ReadMetadataFromFile(fPath) - if err != nil { - return err - } - for _, storageObject := range objMeta.StorageObjects { - err = b.dst.DeleteFileFromObjectDiskBackup(ctx, path.Join(backup.BackupName, diskName, storageObject.ObjectRelativePath)) - if err != nil { - return err - } - } - return nil - }); err != nil { - return err - } - if err := os.RemoveAll(localPath); err != nil { - return err - } - } else if regexp.MustCompile(`/shadow/[^/]+/[^/]+/` + diskName + `/.+$`).MatchString(fName) { - // non compressed remote object disk part - objMetaReader, err := b.dst.GetFileReader(ctx, fName) - if err != nil { - return err - } - objMeta, err := object_disk.ReadMetadataFromReader(objMetaReader, fName) - if err != nil { - return err - } - for _, storageObject := range objMeta.StorageObjects { - err = b.dst.DeleteFileFromObjectDiskBackup(ctx, path.Join(backup.BackupName, diskName, storageObject.ObjectRelativePath)) - if err != nil { - return err - } - } - } + if b.hasObjectDisksRemote(backup) || (b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk == "") { + startTime := time.Now() + if deletedKeys, deleteErr := b.cleanBackupObjectDisks(ctx, backup.BackupName); deleteErr != nil { + log.Warn().Msgf("b.cleanBackupObjectDisks return error: %v", deleteErr) + } else { + log.Info().Str("backup", backup.BackupName).Str("duration", utils.HumanizeDuration(time.Since(startTime))).Msgf("cleanBackupObjectDisks deleted %d keys", deletedKeys) } + return nil } - return nil - }) + } + return nil } -func (b *Backuper) cleanRemoteEmbedded(ctx context.Context, backup storage.Backup, bd *storage.BackupDestination) error { +func (b *Backuper) hasObjectDisksRemote(backup storage.Backup) bool { + for _, diskType := range backup.DiskTypes { + if b.isDiskTypeObject(diskType) { + return true + } + } + return false +} + +func (b *Backuper) cleanRemoteEmbedded(ctx context.Context, backup storage.Backup) error { if err := object_disk.InitCredentialsAndConnections(ctx, b.ch, b.cfg, b.cfg.ClickHouse.EmbeddedBackupDisk); err != nil { return err } - return bd.Walk(ctx, backup.BackupName+"/", true, func(ctx context.Context, f storage.RemoteFile) error { + return b.dst.Walk(ctx, backup.BackupName+"/", true, func(ctx context.Context, f storage.RemoteFile) error { if !strings.HasSuffix(f.Name(), ".json") { - r, err := bd.GetFileReader(ctx, path.Join(backup.BackupName, f.Name())) + r, err := b.dst.GetFileReader(ctx, path.Join(backup.BackupName, f.Name())) if err != nil { return err } - apexLog.Debugf("object_disk.ReadMetadataFromReader(%s)", f.Name()) + log.Debug().Msgf("object_disk.ReadMetadataFromReader(%s)", f.Name()) meta, err := object_disk.ReadMetadataFromReader(r, f.Name()) if err != nil { return err @@ -439,6 +381,29 @@ func (b *Backuper) cleanRemoteEmbedded(ctx context.Context, backup storage.Backu }) } +// cleanBackupObjectDisks - recursive delete / +func (b *Backuper) cleanBackupObjectDisks(ctx context.Context, backupName string) (uint, error) { + objectDiskPath, err := b.getObjectDiskPath() + if err != nil { + return 0, err + } + //walk absolute path, delete relative + deletedKeys := uint(0) + walkErr := b.dst.WalkAbsolute(ctx, path.Join(objectDiskPath, backupName), true, func(ctx context.Context, f storage.RemoteFile) error { + if b.dst.Kind() == "azblob" { + if f.Size() > 0 || !f.LastModified().IsZero() { + deletedKeys += 1 + return b.dst.DeleteFileFromObjectDiskBackup(ctx, path.Join(backupName, f.Name())) + } else { + return nil + } + } + deletedKeys += 1 + return b.dst.DeleteFileFromObjectDiskBackup(ctx, path.Join(backupName, f.Name())) + }) + return deletedKeys, walkErr +} + func (b *Backuper) skipIfSameLocalBackupPresent(ctx context.Context, backupName, tags string) (bool, error) { if localList, _, err := b.GetLocalBackups(ctx, nil); err != nil { return true, err @@ -473,3 +438,20 @@ func (b *Backuper) CleanRemoteBroken(commandId int) error { } return nil } + +func (b *Backuper) cleanPartialRequiredBackup(ctx context.Context, disks []clickhouse.Disk, currentBackupName string) error { + if localBackups, _, err := b.GetLocalBackups(ctx, disks); err == nil { + for _, localBackup := range localBackups { + if localBackup.BackupName != currentBackupName && localBackup.DataSize+localBackup.CompressedSize+localBackup.MetadataSize+localBackup.RBACSize == 0 { + if err = b.RemoveBackupLocal(ctx, localBackup.BackupName, disks); err != nil { + return fmt.Errorf("CleanPartialRequiredBackups %s -> RemoveBackupLocal cleaning error: %v", localBackup.BackupName, err) + } else { + log.Info().Msgf("CleanPartialRequiredBackups %s deleted", localBackup.BackupName) + } + } + } + } else { + return fmt.Errorf("CleanPartialRequiredBackups -> GetLocalBackups cleaning error: %v", err) + } + return nil +} diff --git a/pkg/backup/download.go b/pkg/backup/download.go index 5098ce24..bdb60c13 100644 --- a/pkg/backup/download.go +++ b/pkg/backup/download.go @@ -5,16 +5,18 @@ import ( "encoding/json" "errors" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/custom" - "github.com/Altinity/clickhouse-backup/pkg/filesystemhelper" - "github.com/Altinity/clickhouse-backup/pkg/partition" - "github.com/Altinity/clickhouse-backup/pkg/resumable" - "github.com/Altinity/clickhouse-backup/pkg/status" + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/custom" + "github.com/Altinity/clickhouse-backup/v2/pkg/filesystemhelper" + "github.com/Altinity/clickhouse-backup/v2/pkg/partition" + "github.com/Altinity/clickhouse-backup/v2/pkg/resumable" + "github.com/Altinity/clickhouse-backup/v2/pkg/status" "github.com/eapache/go-resiliency/retrier" + "github.com/rs/zerolog" "io" "io/fs" + "math/rand" "os" "path" "path/filepath" @@ -25,12 +27,12 @@ import ( "time" "golang.org/x/sync/errgroup" - "golang.org/x/sync/semaphore" - "github.com/Altinity/clickhouse-backup/pkg/common" - "github.com/Altinity/clickhouse-backup/pkg/metadata" - "github.com/Altinity/clickhouse-backup/pkg/storage" - "github.com/Altinity/clickhouse-backup/pkg/utils" + "github.com/Altinity/clickhouse-backup/v2/pkg/common" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" + "github.com/rs/zerolog/log" ) @@ -38,39 +40,11 @@ var ( ErrBackupIsAlreadyExists = errors.New("backup is already exists") ) -func (b *Backuper) legacyDownload(ctx context.Context, backupName string) error { - bd, err := storage.NewBackupDestination(ctx, b.cfg, b.ch, true, "") - if err != nil { - return err - } - if err := bd.Connect(ctx); err != nil { - return err - } - defer func() { - if err := bd.Close(ctx); err != nil { - log.Warn().Msgf("can't close BackupDestination error: %v", err) - } - }() - retry := retrier.New(retrier.ConstantBackoff(b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration), nil) - err = retry.RunCtx(ctx, func(ctx context.Context) error { - return bd.DownloadCompressedStream(ctx, backupName, path.Join(b.DefaultDataPath, "backup", backupName)) - }) - if err != nil { - return err - } - log.Info().Fields(map[string]interface{}{ - "backup": backupName, - "operation": "download_legacy", - }).Msg("done") - return nil -} - -func (b *Backuper) Download(backupName string, tablePattern string, partitions []string, schemaOnly, resume bool, commandId int) error { +func (b *Backuper) Download(backupName string, tablePattern string, partitions []string, schemaOnly, resume bool, backupVersion string, commandId int) error { ctx, cancel, err := status.Current.GetContextWithCancel(commandId) if err != nil { return err } - ctx, cancel = context.WithCancel(ctx) defer cancel() backupName = utils.CleanBackupNameRE.ReplaceAllString(backupName, "") if err := b.ch.Connect(); err != nil { @@ -96,12 +70,21 @@ func (b *Backuper) Download(backupName string, tablePattern string, partitions [ if err != nil { return err } + b.DefaultDataPath, err = b.ch.GetDefaultPath(disks) + if err != nil { + return err + } + for i := range localBackups { if backupName == localBackups[i].BackupName { + if strings.Contains(localBackups[i].Tags, "embedded") || b.cfg.General.RemoteStorage == "custom" { + return ErrBackupIsAlreadyExists + } if !b.resume { return ErrBackupIsAlreadyExists } else { - if strings.Contains(localBackups[i].Tags, "embedded") || b.cfg.General.RemoteStorage == "custom" { + _, isResumeExists := os.Stat(path.Join(b.DefaultDataPath, "backup", backupName, "download.state")) + if errors.Is(isResumeExists, os.ErrNotExist) { return ErrBackupIsAlreadyExists } log.Warn().Msgf("%s already exists will try to resume download", backupName) @@ -112,7 +95,7 @@ func (b *Backuper) Download(backupName string, tablePattern string, partitions [ if b.cfg.General.RemoteStorage == "custom" { return custom.Download(ctx, b.cfg, backupName, tablePattern, partitions, schemaOnly) } - if err := b.init(ctx, disks, ""); err != nil { + if err := b.initDisksPathsAndBackupDestination(ctx, disks, ""); err != nil { return err } defer func() { @@ -137,24 +120,13 @@ func (b *Backuper) Download(backupName string, tablePattern string, partitions [ if !found { return fmt.Errorf("'%s' is not found on remote storage", backupName) } - //look https://github.com/Altinity/clickhouse-backup/discussions/266 need download legacy before check for empty backup - if remoteBackup.Legacy { - if tablePattern != "" { - return fmt.Errorf("'%s' is old format backup and doesn't supports download of specific tables", backupName) - } - if schemaOnly { - return fmt.Errorf("'%s' is old format backup and doesn't supports download of schema only", backupName) - } - log.Warn().Msgf("'%s' is old-format backup", backupName) - return b.legacyDownload(ctx, backupName) - } if len(remoteBackup.Tables) == 0 && !b.cfg.General.AllowEmptyBackups { return fmt.Errorf("'%s' is empty backup", backupName) } tablesForDownload := parseTablePatternForDownload(remoteBackup.Tables, tablePattern) if !schemaOnly && !b.cfg.General.DownloadByPart && remoteBackup.RequiredBackup != "" { - err := b.Download(remoteBackup.RequiredBackup, tablePattern, partitions, schemaOnly, b.resume, commandId) + err := b.Download(remoteBackup.RequiredBackup, tablePattern, partitions, schemaOnly, b.resume, backupVersion, commandId) if err != nil && !errors.Is(err, ErrBackupIsAlreadyExists) { return err } @@ -182,26 +154,19 @@ func (b *Backuper) Download(backupName string, tablePattern string, partitions [ } log.Debug().Str("backup", backupName).Msgf("prepare table METADATA concurrent semaphore with concurrency=%d len(tablesForDownload)=%d", b.cfg.General.DownloadConcurrency, len(tablesForDownload)) - tableMetadataAfterDownload := make([]metadata.TableMetadata, len(tablesForDownload)) - downloadSemaphore := semaphore.NewWeighted(int64(b.cfg.General.DownloadConcurrency)) + tableMetadataAfterDownload := make([]*metadata.TableMetadata, len(tablesForDownload)) metadataGroup, metadataCtx := errgroup.WithContext(ctx) + metadataGroup.SetLimit(int(b.cfg.General.DownloadConcurrency)) for i, t := range tablesForDownload { - if err := downloadSemaphore.Acquire(metadataCtx, 1); err != nil { - log.Error().Msgf("can't acquire semaphore during Download metadata: %v", err) - break - } + metadataLogger := log.With().Str("table_metadata", fmt.Sprintf("%s.%s", t.Database, t.Table)).Logger() idx := i tableTitle := t metadataGroup.Go(func() error { - defer downloadSemaphore.Release(1) - downloadedMetadata, size, err := b.downloadTableMetadata(metadataCtx, backupName, disks, tableTitle, schemaOnly, partitions) + downloadedMetadata, size, err := b.downloadTableMetadata(metadataCtx, backupName, disks, tableTitle, schemaOnly, partitions, b.resume, metadataLogger) if err != nil { return err } - if b.shouldSkipByTableEngine(*downloadedMetadata) { - return nil - } - tableMetadataAfterDownload[idx] = *downloadedMetadata + tableMetadataAfterDownload[idx] = downloadedMetadata atomic.AddUint64(&metadataSize, size) return nil }) @@ -209,41 +174,41 @@ func (b *Backuper) Download(backupName string, tablePattern string, partitions [ if err := metadataGroup.Wait(); err != nil { return fmt.Errorf("one of Download Metadata go-routine return error: %v", err) } + // download, missed .inner. tables, https://github.com/Altinity/clickhouse-backup/issues/765 + var missedInnerTableErr error + tableMetadataAfterDownload, tablesForDownload, metadataSize, missedInnerTableErr = b.downloadMissedInnerTablesMetadata(ctx, backupName, metadataSize, tablesForDownload, tableMetadataAfterDownload, disks, schemaOnly, partitions) + if missedInnerTableErr != nil { + return fmt.Errorf("b.downloadMissedInnerTablesMetadata error: %v", missedInnerTableErr) + } + if !schemaOnly { - for _, t := range tableMetadataAfterDownload { - for disk := range t.Parts { - if _, diskExists := b.DiskToPathMap[disk]; !diskExists && disk != b.cfg.ClickHouse.EmbeddedBackupDisk { - b.DiskToPathMap[disk] = b.DiskToPathMap["default"] - log.Warn().Msgf("table '%s.%s' require disk '%s' that not found in clickhouse table system.disks, you can add nonexistent disks to `disk_mapping` in `clickhouse` config section, data will download to %s", t.Database, t.Table, disk, b.DiskToPathMap["default"]) - } - } + if reBalanceErr := b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownload, disks, remoteBackup); reBalanceErr != nil { + return reBalanceErr } - log.Debug().Str("backupName", backupName).Msgf("prepare table SHADOW concurrent semaphore with concurrency=%d len(tableMetadataAfterDownload)=%d", b.cfg.General.DownloadConcurrency, len(tableMetadataAfterDownload)) + log.Debug().Str("backupName", backupName).Msgf("prepare table DATA concurrent semaphore with concurrency=%d len(tableMetadataAfterDownload)=%d", b.cfg.General.DownloadConcurrency, len(tableMetadataAfterDownload)) dataGroup, dataCtx := errgroup.WithContext(ctx) + dataGroup.SetLimit(int(b.cfg.General.DownloadConcurrency)) for i, tableMetadata := range tableMetadataAfterDownload { - if tableMetadata.MetadataOnly { + if tableMetadata == nil || tableMetadata.MetadataOnly { continue } - if err := downloadSemaphore.Acquire(dataCtx, 1); err != nil { - log.Error().Msgf("can't acquire semaphore during Download table data: %v", err) - break - } dataSize += tableMetadata.TotalBytes idx := i dataGroup.Go(func() error { - defer downloadSemaphore.Release(1) start := time.Now() - if err := b.downloadTableData(dataCtx, remoteBackup.BackupMetadata, tableMetadataAfterDownload[idx]); err != nil { + if err := b.downloadTableData(dataCtx, remoteBackup.BackupMetadata, *tableMetadataAfterDownload[idx]); err != nil { return err } - log.Info(). - Str("backup_name", backupName). - Str("operation", "download_data"). - Str("table", fmt.Sprintf("%s.%s", tableMetadataAfterDownload[idx].Database, tableMetadataAfterDownload[idx].Table)). - Str("duration", utils.HumanizeDuration(time.Since(start))). - Str("size", utils.FormatBytes(tableMetadataAfterDownload[idx].TotalBytes)). - Msg("done") + log.Info().Fields(map[string]interface{}{ + "backup_name": backupName, + "operation": "download_data", + "table": fmt.Sprintf("%s.%s", tableMetadataAfterDownload[idx].Database, tableMetadataAfterDownload[idx].Table), + "progress": fmt.Sprintf("%d/%d", idx+1, len(tableMetadataAfterDownload)), + "duration": utils.HumanizeDuration(time.Since(start)), + "size": utils.FormatBytes(tableMetadataAfterDownload[idx].TotalBytes), + "version": backupVersion, + }).Msg("done") return nil }) } @@ -252,39 +217,38 @@ func (b *Backuper) Download(backupName string, tablePattern string, partitions [ } } var rbacSize, configSize uint64 - if !b.isEmbedded { - rbacSize, err = b.downloadRBACData(ctx, remoteBackup) - if err != nil { - return fmt.Errorf("download RBAC error: %v", err) - } + rbacSize, err = b.downloadRBACData(ctx, remoteBackup) + if err != nil { + return fmt.Errorf("download RBAC error: %v", err) + } - configSize, err = b.downloadConfigData(ctx, remoteBackup) - if err != nil { - return fmt.Errorf("download CONFIGS error: %v", err) - } + configSize, err = b.downloadConfigData(ctx, remoteBackup) + if err != nil { + return fmt.Errorf("download CONFIGS error: %v", err) } backupMetadata := remoteBackup.BackupMetadata backupMetadata.Tables = tablesForDownload - backupMetadata.DataSize = dataSize - backupMetadata.MetadataSize = metadataSize - if b.isEmbedded { + if b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk != "" && backupMetadata.Tables != nil && len(backupMetadata.Tables) > 0 { localClickHouseBackupFile := path.Join(b.EmbeddedBackupDataPath, backupName, ".backup") remoteClickHouseBackupFile := path.Join(backupName, ".backup") - if err = b.downloadSingleBackupFile(ctx, remoteClickHouseBackupFile, localClickHouseBackupFile, disks); err != nil { + localEmbeddedMetadataSize := int64(0) + if localEmbeddedMetadataSize, err = b.downloadSingleBackupFile(ctx, remoteClickHouseBackupFile, localClickHouseBackupFile, disks); err != nil { return err } + metadataSize += uint64(localEmbeddedMetadataSize) } backupMetadata.CompressedSize = 0 backupMetadata.DataFormat = "" - backupMetadata.RequiredBackup = "" + backupMetadata.DataSize = dataSize + backupMetadata.MetadataSize = metadataSize backupMetadata.ConfigSize = configSize backupMetadata.RBACSize = rbacSize - + backupMetadata.ClickhouseBackupVersion = backupVersion backupMetafileLocalPath := path.Join(b.DefaultDataPath, "backup", backupName, "metadata.json") - if b.isEmbedded { + if b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk != "" { backupMetafileLocalPath = path.Join(b.EmbeddedBackupDataPath, backupName, "metadata.json") } if err := backupMetadata.Save(backupMetafileLocalPath); err != nil { @@ -302,12 +266,118 @@ func (b *Backuper) Download(backupName string, tablePattern string, partitions [ b.resumableState.Close() } - log.Info(). - Str("operation", "download"). - Str("backup", backupName). - Str("duration", utils.HumanizeDuration(time.Since(startDownload))). - Str("size", utils.FormatBytes(dataSize+metadataSize+rbacSize+configSize)). - Msg("done") + //clean partially downloaded requiredBackup + if remoteBackup.RequiredBackup != "" { + if err = b.cleanPartialRequiredBackup(ctx, disks, remoteBackup.BackupName); err != nil { + return err + } + } + + log.Info().Fields(map[string]interface{}{ + "backup": backupName, + "operation": "download", + "duration": utils.HumanizeDuration(time.Since(startDownload)), + "download_size": utils.FormatBytes(dataSize + metadataSize + rbacSize + configSize), + "object_disk_size": utils.FormatBytes(backupMetadata.ObjectDiskSize), + "version": backupVersion, + }).Msg("done") + return nil +} + +func (b *Backuper) reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownload []*metadata.TableMetadata, disks []clickhouse.Disk, remoteBackup storage.Backup) error { + var disksByStoragePolicyAndType map[string]map[string][]clickhouse.Disk + filterDisksByTypeAndStoragePolicies := func(disk string, diskType string, disks []clickhouse.Disk, remoteBackup storage.Backup, t metadata.TableMetadata) (string, []clickhouse.Disk, error) { + _, ok := remoteBackup.DiskTypes[disk] + if !ok { + return "", nil, fmt.Errorf("disk: %s not found in disk_types section %#v in %s/metadata.json", disk, remoteBackup.DiskTypes, remoteBackup.BackupName) + } + storagePolicy := b.ch.ExtractStoragePolicy(t.Query) + if len(disksByStoragePolicyAndType) == 0 { + disksByStoragePolicyAndType = b.splitDisksByTypeAndStoragePolicy(disks) + } + if _, isTypeExists := disksByStoragePolicyAndType[diskType]; !isTypeExists { + return "", nil, fmt.Errorf("disk: %s, diskType: %s not found in system.disks", disk, diskType) + } + filteredDisks, isPolicyExists := disksByStoragePolicyAndType[diskType][storagePolicy] + if !isPolicyExists || len(filteredDisks) == 0 { + return "", nil, fmt.Errorf("storagePolicy: %s with diskType: %s not found in system.disks", storagePolicy, diskType) + } + return storagePolicy, filteredDisks, nil + } + + updateDiskFreeSize := func(downloadDisk, diskType, storagePolicy string, newFreeSpace uint64) { + for dIdx := range disksByStoragePolicyAndType[diskType][storagePolicy] { + if disksByStoragePolicyAndType[diskType][storagePolicy][dIdx].Name == downloadDisk { + disksByStoragePolicyAndType[diskType][storagePolicy][dIdx].FreeSpace = newFreeSpace + } + } + } + + for i, t := range tableMetadataAfterDownload { + if t == nil || t.TotalBytes == 0 { + continue + } + isRebalanced := false + totalFiles := 0 + for disk := range t.Files { + totalFiles += len(t.Files[disk]) + } + totalParts := 0 + for disk := range t.Parts { + totalParts += len(t.Parts[disk]) + } + if totalFiles == 0 && totalParts == 0 { + continue + } + partSize := t.TotalBytes / uint64(totalParts) + //re-balance parts + for disk := range t.Parts { + if _, diskExists := b.DiskToPathMap[disk]; !diskExists && disk != b.cfg.ClickHouse.EmbeddedBackupDisk { + diskType := remoteBackup.DiskTypes[disk] + storagePolicy, filteredDisks, err := filterDisksByTypeAndStoragePolicies(disk, diskType, disks, remoteBackup, *t) + if err != nil { + return err + } + rebalancedDisks := common.EmptyMap{} + for j := range t.Parts[disk] { + isObjectDisk, downloadDisk, newFreeSpace, reBalanceErr := b.getDownloadDiskForNonExistsDisk(diskType, filteredDisks, partSize) + if reBalanceErr != nil { + return reBalanceErr + } + rebalancedDisks[downloadDisk] = struct{}{} + tableMetadataAfterDownload[i].Parts[disk][j].RebalancedDisk = downloadDisk + isRebalanced = true + if !isObjectDisk { + updateDiskFreeSize(downloadDisk, diskType, storagePolicy, newFreeSpace) + } + //re-balance file depend on part + if t.Files != nil && len(t.Files) > 0 { + if len(t.Files[disk]) == 0 { + return fmt.Errorf("table: `%s`.`%s` part.Name: %s, part.RebalancedDisk: %s, non empty `files` can't find disk: %s", t.Table, t.Database, t.Parts[disk][j].Name, t.Parts[disk][j].RebalancedDisk, disk) + } + for _, fileName := range t.Files[disk] { + if strings.HasPrefix(fileName, disk+"_"+t.Parts[disk][j].Name+".") { + if tableMetadataAfterDownload[i].RebalancedFiles == nil { + tableMetadataAfterDownload[i].RebalancedFiles = map[string]string{} + } + tableMetadataAfterDownload[i].RebalancedFiles[fileName] = downloadDisk + } + } + } + } + rebalancedDisksStr := strings.TrimPrefix( + strings.Replace(fmt.Sprintf("%v", rebalancedDisks), ":{}", "", -1), "map", + ) + log.Warn().Msgf("table '%s.%s' require disk '%s' that not found in system.disks, you can add nonexistent disks to `disk_mapping` in `clickhouse` config section, data will download to %v", t.Database, t.Table, disk, rebalancedDisksStr) + } + } + if isRebalanced { + if _, saveErr := t.Save(t.LocalFile, false); saveErr != nil { + return saveErr + } + } + } + return nil } @@ -318,24 +388,25 @@ func (b *Backuper) downloadTableMetadataIfNotExists(ctx context.Context, backupN return tm, nil } // we always download full metadata in this case without filter by partitions - tm, _, err := b.downloadTableMetadata(ctx, backupName, nil, tableTitle, false, nil) + logger := log.With().Fields(map[string]interface{}{"operation": "downloadTableMetadataIfNotExists", "backupName": backupName, "table_metadata_diff": fmt.Sprintf("%s.%s", tableTitle.Database, tableTitle.Table)}).Logger() + tm, _, err := b.downloadTableMetadata(ctx, backupName, nil, tableTitle, false, nil, false, logger) return tm, err } -func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string, disks []clickhouse.Disk, tableTitle metadata.TableTitle, schemaOnly bool, partitions []string) (*metadata.TableMetadata, uint64, error) { +func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string, disks []clickhouse.Disk, tableTitle metadata.TableTitle, schemaOnly bool, partitions []string, resume bool, logger zerolog.Logger) (*metadata.TableMetadata, uint64, error) { start := time.Now() size := uint64(0) metadataFiles := map[string]string{} remoteMedataPrefix := path.Join(backupName, "metadata", common.TablePathEncode(tableTitle.Database), common.TablePathEncode(tableTitle.Table)) metadataFiles[fmt.Sprintf("%s.json", remoteMedataPrefix)] = path.Join(b.DefaultDataPath, "backup", backupName, "metadata", common.TablePathEncode(tableTitle.Database), fmt.Sprintf("%s.json", common.TablePathEncode(tableTitle.Table))) partitionsIdMap := make(map[metadata.TableTitle]common.EmptyMap) - if b.isEmbedded { + if b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk != "" { metadataFiles[fmt.Sprintf("%s.sql", remoteMedataPrefix)] = path.Join(b.EmbeddedBackupDataPath, backupName, "metadata", common.TablePathEncode(tableTitle.Database), fmt.Sprintf("%s.sql", common.TablePathEncode(tableTitle.Table))) metadataFiles[fmt.Sprintf("%s.json", remoteMedataPrefix)] = path.Join(b.EmbeddedBackupDataPath, backupName, "metadata", common.TablePathEncode(tableTitle.Database), fmt.Sprintf("%s.json", common.TablePathEncode(tableTitle.Table))) } var tableMetadata metadata.TableMetadata for remoteMetadataFile, localMetadataFile := range metadataFiles { - if b.resume { + if resume { isProcessed, processedSize := b.resumableState.IsAlreadyProcessed(localMetadataFile) if isProcessed && strings.HasSuffix(localMetadataFile, ".json") { tmBody, err := os.ReadFile(localMetadataFile) @@ -347,6 +418,7 @@ func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string, } partitionsIdMap, _ = partition.ConvertPartitionsToIdsMapAndNamesList(ctx, b.ch, nil, []metadata.TableMetadata{tableMetadata}, partitions) filterPartsAndFilesByPartitionsFilter(tableMetadata, partitionsIdMap[metadata.TableTitle{Database: tableMetadata.Database, Table: tableMetadata.Table}]) + tableMetadata.LocalFile = localMetadataFile } if isProcessed { size += uint64(processedSize) @@ -391,6 +463,9 @@ func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string, if err = json.Unmarshal(tmBody, &tableMetadata); err != nil { return nil, 0, err } + if b.shouldSkipByTableEngine(tableMetadata) || b.shouldSkipByTableName(fmt.Sprintf("%s.%s", tableMetadata.Database, tableMetadata.Table)) { + return nil, 0, nil + } partitionsIdMap, _ = partition.ConvertPartitionsToIdsMapAndNamesList(ctx, b.ch, nil, []metadata.TableMetadata{tableMetadata}, partitions) filterPartsAndFilesByPartitionsFilter(tableMetadata, partitionsIdMap[metadata.TableTitle{Database: tableMetadata.Database, Table: tableMetadata.Table}]) // save metadata @@ -401,12 +476,13 @@ func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string, } written = int64(jsonSize) size += jsonSize + tableMetadata.LocalFile = localMetadataFile } - if b.resume { + if resume { b.resumableState.AppendToState(localMetadataFile, written) } } - log.Info().Fields(map[string]string{ + logger.Info().Fields(map[string]string{ "operation": "download_metadata", "backup": backupName, "duration": utils.HumanizeDuration(time.Since(start)), @@ -415,6 +491,49 @@ func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string, return &tableMetadata, size, nil } +// downloadMissedInnerTablesMetadata - download, missed .inner. tables if materialized view query not contains `TO db.table` clause, https://github.com/Altinity/clickhouse-backup/issues/765 +// @todo think about parallel download if sequentially will slow +func (b *Backuper) downloadMissedInnerTablesMetadata(ctx context.Context, backupName string, metadataSize uint64, tablesForDownload []metadata.TableTitle, tableMetadataAfterDownload []*metadata.TableMetadata, disks []clickhouse.Disk, schemaOnly bool, partitions []string) ([]*metadata.TableMetadata, []metadata.TableTitle, uint64, error) { + if b.isEmbedded { + return tableMetadataAfterDownload, tablesForDownload, metadataSize, nil + } + for _, t := range tableMetadataAfterDownload { + if t == nil { + continue + } + if strings.HasPrefix(t.Query, "ATTACH MATERIALIZED") || strings.HasPrefix(t.Query, "CREATE MATERIALIZED") { + if strings.Contains(t.Query, " TO ") && !strings.Contains(t.Query, " TO INNER UUID") { + continue + } + var innerTableName string + if matches := uuidRE.FindStringSubmatch(t.Query); len(matches) > 0 { + innerTableName = fmt.Sprintf(".inner_id.%s", matches[1]) + } else { + innerTableName = fmt.Sprintf(".inner.%s", t.Table) + } + innerTableExists := false + for _, existsTable := range tablesForDownload { + if existsTable.Table == innerTableName && existsTable.Database == t.Database { + innerTableExists = true + break + } + } + if !innerTableExists { + innerTableTitle := metadata.TableTitle{Database: t.Database, Table: innerTableName} + metadataLogger := log.With().Str("missed_inner_metadata", fmt.Sprintf("%s.%s", innerTableTitle.Database, innerTableTitle.Table)).Logger() + innerTableMetadata, size, err := b.downloadTableMetadata(ctx, backupName, disks, innerTableTitle, schemaOnly, partitions, b.resume, metadataLogger) + if err != nil { + return tableMetadataAfterDownload, tablesForDownload, metadataSize, err + } + metadataSize += size + tablesForDownload = append(tablesForDownload, innerTableTitle) + tableMetadataAfterDownload = append(tableMetadataAfterDownload, innerTableMetadata) + } + } + } + return tableMetadataAfterDownload, tablesForDownload, metadataSize, nil +} + func (b *Backuper) downloadRBACData(ctx context.Context, remoteBackup storage.Backup) (uint64, error) { return b.downloadBackupRelatedDir(ctx, remoteBackup, "access") } @@ -427,7 +546,7 @@ func (b *Backuper) downloadBackupRelatedDir(ctx context.Context, remoteBackup st localDir := path.Join(b.DefaultDataPath, "backup", remoteBackup.BackupName, prefix) if remoteBackup.DataFormat != DirectoryFormat { - prefix = fmt.Sprintf("%s.%s", prefix, b.cfg.GetArchiveExtension()) + prefix = fmt.Sprintf("%s.%s", prefix, config.ArchiveExtensions[remoteBackup.DataFormat]) } remoteSource := path.Join(remoteBackup.BackupName, prefix) @@ -437,7 +556,7 @@ func (b *Backuper) downloadBackupRelatedDir(ctx context.Context, remoteBackup st } } if remoteBackup.DataFormat == DirectoryFormat { - if err := b.dst.DownloadPath(ctx, 0, remoteSource, localDir, b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration); err != nil { + if err := b.dst.DownloadPath(ctx, remoteSource, localDir, b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration, b.cfg.General.DownloadMaxBytesPerSecond); err != nil { //SFTP can't walk on non exists paths and return error if !strings.Contains(err.Error(), "not exist") { return 0, err @@ -471,7 +590,7 @@ func (b *Backuper) downloadBackupRelatedDir(ctx context.Context, remoteBackup st } retry := retrier.New(retrier.ConstantBackoff(b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration), nil) err = retry.RunCtx(ctx, func(ctx context.Context) error { - return b.dst.DownloadCompressedStream(ctx, remoteSource, localDir) + return b.dst.DownloadCompressedStream(ctx, remoteSource, localDir, b.cfg.General.DownloadMaxBytesPerSecond) }) if err != nil { return 0, err @@ -484,9 +603,10 @@ func (b *Backuper) downloadBackupRelatedDir(ctx context.Context, remoteBackup st func (b *Backuper) downloadTableData(ctx context.Context, remoteBackup metadata.BackupMetadata, table metadata.TableMetadata) error { dbAndTableDir := path.Join(common.TablePathEncode(table.Database), common.TablePathEncode(table.Table)) - - s := semaphore.NewWeighted(int64(b.cfg.General.DownloadConcurrency)) - g, dataCtx := errgroup.WithContext(ctx) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + dataGroup, dataCtx := errgroup.WithContext(ctx) + dataGroup.SetLimit(int(b.cfg.General.DownloadConcurrency)) if remoteBackup.DataFormat != DirectoryFormat { capacity := 0 @@ -496,29 +616,28 @@ func (b *Backuper) downloadTableData(ctx context.Context, remoteBackup metadata. downloadOffset[disk] = 0 } log.Debug().Msgf("start %s.%s with concurrency=%d len(table.Files[...])=%d", table.Database, table.Table, b.cfg.General.DownloadConcurrency, capacity) - breakByErrorArchive: for common.SumMapValuesInt(downloadOffset) < capacity { for disk := range table.Files { if downloadOffset[disk] >= len(table.Files[disk]) { continue } archiveFile := table.Files[disk][downloadOffset[disk]] - if err := s.Acquire(dataCtx, 1); err != nil { - log.Error().Msgf("can't acquire semaphore %s archive: %v", archiveFile, err) - break breakByErrorArchive + diskName := disk + isRebalanced := false + if diskName, isRebalanced = table.RebalancedFiles[archiveFile]; !isRebalanced { + diskName = disk } - tableLocalDir := b.getLocalBackupDataPathForTable(remoteBackup.BackupName, disk, dbAndTableDir) + tableLocalDir := b.getLocalBackupDataPathForTable(remoteBackup.BackupName, diskName, dbAndTableDir) downloadOffset[disk] += 1 tableRemoteFile := path.Join(remoteBackup.BackupName, "shadow", common.TablePathEncode(table.Database), common.TablePathEncode(table.Table), archiveFile) - g.Go(func() error { - defer s.Release(1) + dataGroup.Go(func() error { log.Debug().Msgf("start download %s", tableRemoteFile) if b.resume && b.resumableState.IsAlreadyProcessedBool(tableRemoteFile) { return nil } retry := retrier.New(retrier.ConstantBackoff(b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration), nil) err := retry.RunCtx(dataCtx, func(dataCtx context.Context) error { - return b.dst.DownloadCompressedStream(dataCtx, tableRemoteFile, tableLocalDir) + return b.dst.DownloadCompressedStream(dataCtx, tableRemoteFile, tableLocalDir, b.cfg.General.DownloadMaxBytesPerSecond) }) if err != nil { return err @@ -538,10 +657,9 @@ func (b *Backuper) downloadTableData(ctx context.Context, remoteBackup metadata. } log.Debug().Msgf("start %s.%s with concurrency=%d len(table.Parts[...])=%d", table.Database, table.Table, b.cfg.General.DownloadConcurrency, capacity) - breakByErrorDirectory: for disk, parts := range table.Parts { tableRemotePath := path.Join(remoteBackup.BackupName, "shadow", dbAndTableDir, disk) - diskPath := b.DiskToPathMap[disk] + diskPath, diskExists := b.DiskToPathMap[disk] tableLocalPath := path.Join(diskPath, "backup", remoteBackup.BackupName, "shadow", dbAndTableDir, disk) if b.isEmbedded { tableLocalPath = path.Join(diskPath, remoteBackup.BackupName, "data", dbAndTableDir) @@ -550,19 +668,20 @@ func (b *Backuper) downloadTableData(ctx context.Context, remoteBackup metadata. if part.Required { continue } - partRemotePath := path.Join(tableRemotePath, part.Name) - if err := s.Acquire(dataCtx, 1); err != nil { - log.Error().Msgf("can't acquire semaphore %s directory: %v", partRemotePath, err) - break breakByErrorDirectory + if !diskExists { + diskPath, diskExists = b.DiskToPathMap[part.RebalancedDisk] + if !diskExists { + return fmt.Errorf("downloadTableData: table: `%s`.`%s`, disk: %s, part.Name: %s, part.RebalancedDisk: %s not rebalanced", table.Table, table.Database, disk, part.Name, part.RebalancedDisk) + } } + partRemotePath := path.Join(tableRemotePath, part.Name) partLocalPath := path.Join(tableLocalPath, part.Name) - g.Go(func() error { - defer s.Release(1) + dataGroup.Go(func() error { log.Debug().Msgf("start %s -> %s", partRemotePath, partLocalPath) if b.resume && b.resumableState.IsAlreadyProcessedBool(partRemotePath) { return nil } - if err := b.dst.DownloadPath(dataCtx, 0, partRemotePath, partLocalPath, b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration); err != nil { + if err := b.dst.DownloadPath(dataCtx, partRemotePath, partLocalPath, b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration, b.cfg.General.DownloadMaxBytesPerSecond); err != nil { return err } if b.resume { @@ -574,11 +693,11 @@ func (b *Backuper) downloadTableData(ctx context.Context, remoteBackup metadata. } } } - if err := g.Wait(); err != nil { + if err := dataGroup.Wait(); err != nil { return fmt.Errorf("one of downloadTableData go-routine return error: %v", err) } - if !b.isEmbedded { + if !b.isEmbedded && remoteBackup.RequiredBackup != "" { err := b.downloadDiffParts(ctx, remoteBackup, table, dbAndTableDir) if err != nil { return err @@ -594,17 +713,28 @@ func (b *Backuper) downloadDiffParts(ctx context.Context, remoteBackup metadata. Str("table", fmt.Sprintf("%s.%s", table.Database, table.Table)). Msg("start") start := time.Now() + ctx, cancel := context.WithCancel(ctx) + defer cancel() downloadedDiffParts := uint32(0) - s := semaphore.NewWeighted(int64(b.cfg.General.DownloadConcurrency)) downloadDiffGroup, downloadDiffCtx := errgroup.WithContext(ctx) - + downloadDiffGroup.SetLimit(int(b.cfg.General.DownloadConcurrency)) diffRemoteFilesCache := map[string]*sync.Mutex{} diffRemoteFilesLock := &sync.Mutex{} -breakByError: for disk, parts := range table.Parts { + diskPath, diskExists := b.DiskToPathMap[disk] for _, part := range parts { - newPath := path.Join(b.DiskToPathMap[disk], "backup", remoteBackup.BackupName, "shadow", dbAndTableDir, disk, part.Name) + if !diskExists && part.RebalancedDisk == "" { + return fmt.Errorf("downloadDiffParts: table: `%s`.`%s`, disk: %s, part.Name: %s, part.RebalancedDisk: `%s` not rebalanced", table.Table, table.Database, disk, part.Name, part.RebalancedDisk) + } + if !diskExists { + diskPath, diskExists = b.DiskToPathMap[part.RebalancedDisk] + if !diskExists { + return fmt.Errorf("downloadDiffParts: table: `%s`.`%s`, disk: %s, part.Name: %s, part.RebalancedDisk: `%s` not rebalanced", table.Table, table.Database, disk, part.Name, part.RebalancedDisk) + } + disk = part.RebalancedDisk + } + newPath := path.Join(diskPath, "backup", remoteBackup.BackupName, "shadow", dbAndTableDir, disk, part.Name) if err := b.checkNewPath(newPath, part); err != nil { return err } @@ -617,14 +747,23 @@ breakByError: return fmt.Errorf("%s stat return error: %v", existsPath, err) } if err != nil && os.IsNotExist(err) { - if err := s.Acquire(downloadDiffCtx, 1); err != nil { - log.Error().Msgf("can't acquire semaphore during downloadDiffParts: %v", err) - break breakByError + //if existPath already processed then expect non empty newPath + if b.resume && b.resumableState.IsAlreadyProcessedBool(existsPath) { + if newPathDirList, newPathDirErr := os.ReadDir(newPath); newPathDirErr != nil { + newPathDirErr = fmt.Errorf("os.ReadDir(%s) error: %v", newPath, newPathDirErr) + log.Error().Msg(newPathDirErr.Error()) + return newPathDirErr + } else if len(newPathDirList) == 0 { + return fmt.Errorf("os.ReadDir(%s) expect return non empty list", newPath) + } + continue } partForDownload := part diskForDownload := disk + if !diskExists { + diskForDownload = part.RebalancedDisk + } downloadDiffGroup.Go(func() error { - defer s.Release(1) tableRemoteFiles, err := b.findDiffBackupFilesRemote(downloadDiffCtx, remoteBackup, table, diskForDownload, partForDownload) if err != nil { return err @@ -643,11 +782,11 @@ breakByError: return fmt.Errorf("after downloadDiffRemoteFile %s exists but is not directory", downloadedPartPath) } if err = b.makePartHardlinks(downloadedPartPath, existsPath); err != nil { - return fmt.Errorf("can't to add link to exists part %s -> %s error: %v", newPath, existsPath, err) + return fmt.Errorf("can't to add link to rebalanced part %s -> %s error: %v", downloadedPartPath, existsPath, err) } } if err != nil && !os.IsNotExist(err) { - return fmt.Errorf("after downloadDiffRemoteFile %s stat return error: %v", downloadedPartPath, err) + return fmt.Errorf("after downloadDiffRemoteFile os.Stat(%s) return error: %v", downloadedPartPath, err) } } atomic.AddUint32(&downloadedDiffParts, 1) @@ -682,6 +821,9 @@ breakByError: } func (b *Backuper) downloadDiffRemoteFile(ctx context.Context, diffRemoteFilesLock *sync.Mutex, diffRemoteFilesCache map[string]*sync.Mutex, tableRemoteFile string, tableLocalDir string) error { + if b.resume && b.resumableState.IsAlreadyProcessedBool(tableRemoteFile) { + return nil + } diffRemoteFilesLock.Lock() namedLock, isCached := diffRemoteFilesCache[tableRemoteFile] if isCached { @@ -699,7 +841,7 @@ func (b *Backuper) downloadDiffRemoteFile(ctx context.Context, diffRemoteFilesLo if path.Ext(tableRemoteFile) != "" { retry := retrier.New(retrier.ConstantBackoff(b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration), nil) err := retry.RunCtx(ctx, func(ctx context.Context) error { - return b.dst.DownloadCompressedStream(ctx, tableRemoteFile, tableLocalDir) + return b.dst.DownloadCompressedStream(ctx, tableRemoteFile, tableLocalDir, b.cfg.General.DownloadMaxBytesPerSecond) }) if err != nil { log.Warn().Msgf("DownloadCompressedStream %s -> %s return error: %v", tableRemoteFile, tableLocalDir, err) @@ -707,12 +849,15 @@ func (b *Backuper) downloadDiffRemoteFile(ctx context.Context, diffRemoteFilesLo } } else { // remoteFile could be a directory - if err := b.dst.DownloadPath(ctx, 0, tableRemoteFile, tableLocalDir, b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration); err != nil { + if err := b.dst.DownloadPath(ctx, tableRemoteFile, tableLocalDir, b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration, b.cfg.General.DownloadMaxBytesPerSecond); err != nil { log.Warn().Msgf("DownloadPath %s -> %s return error: %v", tableRemoteFile, tableLocalDir, err) return err } } namedLock.Unlock() + if b.resume { + b.resumableState.AppendToState(tableRemoteFile, 0) + } log.Debug().Str("tableRemoteFile", tableRemoteFile).Msgf("finish download") } return nil @@ -854,17 +999,22 @@ func (b *Backuper) findDiffFileExist(ctx context.Context, requiredBackup *metada log.Debug().Fields(map[string]interface{}{"tableRemoteFile": tableRemoteFile, "tableRemotePath": tableRemotePath, "part": part.Name}).Msg("findDiffFileExist not found") return "", "", err } - if tableLocalDir, diskExists := b.DiskToPathMap[localDisk]; !diskExists { - return "", "", fmt.Errorf("`%s` is not found in system.disks", localDisk) - } else { - if path.Ext(tableRemoteFile) == ".txt" { - tableLocalDir = path.Join(tableLocalDir, "backup", requiredBackup.BackupName, "shadow", dbAndTableDir, localDisk, part.Name) - } else { - tableLocalDir = path.Join(tableLocalDir, "backup", requiredBackup.BackupName, "shadow", dbAndTableDir, localDisk) + tableLocalDir, diskExists := b.DiskToPathMap[localDisk] + if !diskExists { + tableLocalDir, diskExists = b.DiskToPathMap[part.RebalancedDisk] + if !diskExists { + return "", "", fmt.Errorf("localDisk:%s, part.Name: %s, part.RebalancedDisk: %s is not found in system.disks", localDisk, part.Name, part.RebalancedDisk) } - log.Debug().Fields(map[string]interface{}{"tableRemoteFile": tableRemoteFile, "tableRemotePath": tableRemotePath, "part": part.Name}).Msg("findDiffFileExist found") - return tableRemotePath, tableLocalDir, nil + localDisk = part.RebalancedDisk } + + if path.Ext(tableRemoteFile) == ".txt" { + tableLocalDir = path.Join(tableLocalDir, "backup", requiredBackup.BackupName, "shadow", dbAndTableDir, localDisk, part.Name) + } else { + tableLocalDir = path.Join(tableLocalDir, "backup", requiredBackup.BackupName, "shadow", dbAndTableDir, localDisk) + } + log.Debug().Fields(map[string]interface{}{"tableRemoteFile": tableRemoteFile, "tableRemotePath": tableRemotePath, "part": part.Name}).Msg("findDiffFileExist found") + return tableRemotePath, tableLocalDir, nil } func (b *Backuper) ReadBackupMetadataRemote(ctx context.Context, backupName string) (*metadata.BackupMetadata, error) { @@ -881,43 +1031,55 @@ func (b *Backuper) ReadBackupMetadataRemote(ctx context.Context, backupName stri } func (b *Backuper) makePartHardlinks(exists, new string) error { - ex, err := os.Open(exists) - if err != nil { - return err - } - defer func() { - if err = ex.Close(); err != nil { - log.Warn().Msgf("Can't close %s", exists) - } - }() - files, err := ex.Readdirnames(-1) + _, err := os.Stat(exists) if err != nil { return err } - if err := os.MkdirAll(new, 0750); err != nil { + if err = os.MkdirAll(new, 0750); err != nil { log.Warn().Msgf("MkDirAll(%s) error: %v", new, err) return err } - for _, f := range files { - existsF := path.Join(exists, f) - newF := path.Join(new, f) - if err := os.Link(existsF, newF); err != nil { + if walkErr := filepath.Walk(exists, func(fPath string, fInfo os.FileInfo, err error) error { + if err != nil { + return err + } + fPath = strings.TrimPrefix(fPath, exists) + existsF := path.Join(exists, fPath) + newF := path.Join(new, fPath) + if fInfo.IsDir() { + if err = os.MkdirAll(newF, fInfo.Mode()); err != nil { + log.Warn().Msgf("MkdirAll(%s) error: %v", fPath, err) + return err + } + return nil + } + + if err = os.Link(existsF, newF); err != nil { existsFInfo, existsStatErr := os.Stat(existsF) newFInfo, newStatErr := os.Stat(newF) if existsStatErr != nil || newStatErr != nil || !os.SameFile(existsFInfo, newFInfo) { - log.Warn().Msgf("Link %s -> %s error: %v", newF, existsF, err) + log.Warn().Msgf("Link %s -> %s error: %v, existsStatErr: %v newStatErr: %v", existsF, newF, err, existsStatErr, newStatErr) return err } } + return nil + }); walkErr != nil { + log.Warn().Msgf("Link recursively %s -> %s return error: %v", new, exists, walkErr) + return walkErr } return nil } -func (b *Backuper) downloadSingleBackupFile(ctx context.Context, remoteFile string, localFile string, disks []clickhouse.Disk) error { - if b.resume && b.resumableState.IsAlreadyProcessedBool(remoteFile) { - return nil +func (b *Backuper) downloadSingleBackupFile(ctx context.Context, remoteFile string, localFile string, disks []clickhouse.Disk) (int64, error) { + var size int64 + var isProcessed bool + if b.resume { + if isProcessed, size = b.resumableState.IsAlreadyProcessed(remoteFile); isProcessed { + return size, nil + } } retry := retrier.New(retrier.ConstantBackoff(b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration), nil) + err := retry.RunCtx(ctx, func(ctx context.Context) error { remoteReader, err := b.dst.GetFileReader(ctx, remoteFile) if err != nil { @@ -941,7 +1103,7 @@ func (b *Backuper) downloadSingleBackupFile(ctx context.Context, remoteFile stri } }() - _, err = io.CopyBuffer(localWriter, remoteReader, nil) + size, err = io.CopyBuffer(localWriter, remoteReader, nil) if err != nil { return err } @@ -952,10 +1114,53 @@ func (b *Backuper) downloadSingleBackupFile(ctx context.Context, remoteFile stri return nil }) if err != nil { - return err + return 0, err } if b.resume { - b.resumableState.AppendToState(remoteFile, 0) + b.resumableState.AppendToState(remoteFile, size) } - return nil + return size, nil +} + +// filterDisksByStoragePolicyAndType - https://github.com/Altinity/clickhouse-backup/issues/561 +func (b *Backuper) splitDisksByTypeAndStoragePolicy(disks []clickhouse.Disk) map[string]map[string][]clickhouse.Disk { + disksByTypeAndPolicy := map[string]map[string][]clickhouse.Disk{} + for _, d := range disks { + if !d.IsBackup { + if _, typeExists := disksByTypeAndPolicy[d.Type]; !typeExists { + disksByTypeAndPolicy[d.Type] = map[string][]clickhouse.Disk{} + } + for _, policy := range d.StoragePolicies { + if _, policyExists := disksByTypeAndPolicy[d.Type][policy]; !policyExists { + disksByTypeAndPolicy[d.Type][policy] = []clickhouse.Disk{} + } + disksByTypeAndPolicy[d.Type][policy] = append(disksByTypeAndPolicy[d.Type][policy], d) + } + } + } + return disksByTypeAndPolicy +} + +// getDownloadDiskForNonExistsDisk - https://github.com/Altinity/clickhouse-backup/issues/561 +// allow to Restore to new server with different storage policy, different disk count, +// implements `least_used` for normal disk and `random` for Object disks +func (b *Backuper) getDownloadDiskForNonExistsDisk(notExistsDiskType string, filteredDisks []clickhouse.Disk, partSize uint64) (bool, string, uint64, error) { + // random for non-local disks + if notExistsDiskType != "local" { + roundRobinIdx := rand.Intn(len(filteredDisks)) + return true, filteredDisks[roundRobinIdx].Name, 0, nil + } + // least_used for local + freeSpace := partSize + leastUsedIdx := -1 + for idx, d := range filteredDisks { + if filteredDisks[idx].FreeSpace > freeSpace { + freeSpace = d.FreeSpace + leastUsedIdx = idx + } + } + if leastUsedIdx < 0 { + return false, "", 0, fmt.Errorf("%s free space, not found in system.disks with `local` type", utils.FormatBytes(partSize)) + } + return false, filteredDisks[leastUsedIdx].Name, filteredDisks[leastUsedIdx].FreeSpace - partSize, nil } diff --git a/pkg/backup/download_test.go b/pkg/backup/download_test.go new file mode 100644 index 00000000..2e93324b --- /dev/null +++ b/pkg/backup/download_test.go @@ -0,0 +1,286 @@ +package backup + +import ( + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage" + "github.com/stretchr/testify/assert" + "regexp" + "testing" + "time" +) + +var b = Backuper{ + DefaultDataPath: "/var/lib/clickhouse", + DiskToPathMap: map[string]string{ + "default": "/var/lib/clickhouse", + "s3": "/var/lib/clickhouse/disks/s3", + }, + cfg: &config.Config{ClickHouse: config.ClickHouseConfig{EmbeddedBackupDisk: "backup"}}, + isEmbedded: false, +} + +var baseDisks = []clickhouse.Disk{ + { + Name: "default", + Path: "/var/lib/clickhouse", + Type: "local", + FreeSpace: 1024, + StoragePolicies: []string{"default", "jbod"}, + IsBackup: false, + }, + { + Name: "s3", + Path: "/var/lib/clickhouse/disks/s3", + Type: "s3", + FreeSpace: 0, + StoragePolicies: []string{"s3_only", "jbod"}, + IsBackup: false, + }, + { + Name: "backup_s3", + Path: "/var/lib/clickhouse/disks/backup_s3", + Type: "s3", + FreeSpace: 0, + StoragePolicies: nil, + IsBackup: true, + }, +} +var jbodDisks = []clickhouse.Disk{ + { + Name: "hdd2", + Path: "/hdd2", + Type: "local", + FreeSpace: 1024, + StoragePolicies: []string{"default", "jbod"}, + IsBackup: false, + }, + { + Name: "s3_disk2", + Path: "/var/lib/clickhouse/disks/s3_disk2", + Type: "s3", + FreeSpace: 0, + StoragePolicies: []string{"s3_only", "jbod"}, + IsBackup: false, + }, +} + +var remoteBackup = storage.Backup{ + BackupMetadata: metadata.BackupMetadata{ + BackupName: "Test", + // doesn't matter which disk path in backup, should use system.disks? + Disks: map[string]string{"default": "/test", "hdd2": "/disk2", "s3": "/s3", "s3_disk2": "/s3_new"}, + DiskTypes: map[string]string{"default": "local", "hdd2": "local", "s3": "s3", "s3_disk2": "s3"}, + ClickhouseBackupVersion: "unknown", + CreationDate: time.Now(), + Tags: "", + ClickHouseVersion: "unknown", + Databases: nil, + Tables: []metadata.TableTitle{ + { + Database: "default", + Table: "test", + }, + }, + Functions: nil, + DataFormat: "tar", + RequiredBackup: "", + }, + UploadDate: time.Now(), +} + +func TestReBalanceTablesMetadataIfDiskNotExists_Files_NoErrors(t *testing.T) { + remoteBackup.DataFormat = "tar" + baseTable := metadata.TableMetadata{ + Files: map[string][]string{ + "default": {"default_part_1_1_0.tar", "default_part_2_2_0.tar"}, + "s3": {"s3_part_5_5_0.tar", "s3_part_6_6_0.tar"}, + }, + Parts: map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "s3": {{Name: "part_5_5_0"}, {Name: "part_6_6_0"}}, + }, + RebalancedFiles: nil, + Database: "default", + Table: "test", + Query: "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id", + Size: nil, + TotalBytes: 1000, + Mutations: nil, + MetadataOnly: false, + LocalFile: "/dev/null", + } + var tableMetadataAfterDownload []metadata.TableMetadata + tableMetadataAfterDownload = append(tableMetadataAfterDownload, baseTable) + baseTable.Table = "test2" + baseTable.Query = "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='jbod'" + baseTable.Files = map[string][]string{ + "default": {"default_part_1_1_0.tar", "default_part_2_2_0.tar"}, + "hdd2": {"hdd2_part_3_3_0.tar", "hdd2_part_4_4_0.tar"}, + "s3": {"s3_part_5_5_0.tar", "s3_part_6_6_0.tar"}, + "s3_disk2": {"s3_disk2_part_7_7_0.tar", "s3_disk2_part_8_8_0.tar"}, + } + baseTable.Parts = map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "hdd2": {{Name: "part_3_3_0"}, {Name: "part_4_4_0"}}, + "s3": {{Name: "part_5_5_0"}, {Name: "part_6_6_0"}}, + "s3_disk2": {{Name: "part_7_7_0"}, {Name: "part_8_8_0"}}, + } + + tableMetadataAfterDownload = append(tableMetadataAfterDownload, baseTable) + tableMetadataAfterDownloadRepacked := make([]*metadata.TableMetadata, len(tableMetadataAfterDownload)) + for i := range tableMetadataAfterDownload { + tableMetadataAfterDownloadRepacked[i] = &tableMetadataAfterDownload[i] + } + assert.NoError(t, b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownloadRepacked, baseDisks, remoteBackup)) + //rebalanced table + meta := tableMetadataAfterDownload[1] + assert.Equal(t, 4, len(meta.RebalancedFiles), "expect 4 rebalanced files in %s.%s", meta.Database, meta.Table) + for i, d := range jbodDisks { + for _, p := range meta.Parts[d.Name] { + assert.Equal(t, baseDisks[i].Name, p.RebalancedDisk, "expect rebalanced part:%s", p.Name) + } + } + + //non-rebalanced table + meta = tableMetadataAfterDownload[0] + assert.Equal(t, 0, len(meta.RebalancedFiles), "0 files shall rebalanced") + for _, d := range baseDisks { + for _, p := range meta.Parts[d.Name] { + assert.Equal(t, "", p.RebalancedDisk, "expect no rebalanced part: %s", p.Name) + } + } + +} + +func TestReBalanceTablesMetadataIfDiskNotExists_Parts_NoErrors(t *testing.T) { + remoteBackup.DataFormat = "directory" + baseTable := metadata.TableMetadata{ + Parts: map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "s3": {{Name: "part_5_5_0"}, {Name: "part_6_6_0"}}, + }, + RebalancedFiles: nil, + Database: "default", + Table: "test", + Query: "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id", + Size: nil, + TotalBytes: 1000, + Mutations: nil, + MetadataOnly: false, + LocalFile: "/dev/null", + } + var tableMetadataAfterDownload []metadata.TableMetadata + tableMetadataAfterDownload = append(tableMetadataAfterDownload, baseTable) + baseTable.Table = "test2" + baseTable.Query = "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='jbod'" + baseTable.Parts = map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "hdd2": {{Name: "part_3_3_0"}, {Name: "part_4_4_0"}}, + "s3": {{Name: "part_5_5_0"}, {Name: "part_6_6_0"}}, + "s3_disk2": {{Name: "part_7_7_0"}, {Name: "part_8_8_0"}}, + } + + tableMetadataAfterDownload = append(tableMetadataAfterDownload, baseTable) + + tableMetadataAfterDownloadRepacked := make([]*metadata.TableMetadata, len(tableMetadataAfterDownload)) + for i := range tableMetadataAfterDownload { + tableMetadataAfterDownloadRepacked[i] = &tableMetadataAfterDownload[i] + } + assert.NoError(t, b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownloadRepacked, baseDisks, remoteBackup)) + // no files re-balance + for _, meta := range tableMetadataAfterDownload { + assert.Equal(t, 0, len(meta.RebalancedFiles)) + } + // re-balance parts + meta := tableMetadataAfterDownload[1] + for i, d := range jbodDisks { + for _, p := range meta.Parts[d.Name] { + assert.Equal(t, baseDisks[i].Name, p.RebalancedDisk, "expect rebalanced part") + } + } + // non re-balance parts + meta = tableMetadataAfterDownload[0] + for _, d := range baseDisks { + for _, p := range meta.Parts[d.Name] { + assert.Equal(t, "", p.RebalancedDisk, "expect no rebalanced part") + } + } +} + +func TestReBalanceTablesMetadataIfDiskNotExists_CheckErrors(t *testing.T) { + invalidRemoteBackup := remoteBackup + invalidRemoteBackup.DataFormat = DirectoryFormat + invalidTable := metadata.TableMetadata{ + Parts: map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "hdd2": {{Name: "part_3_3_0"}, {Name: "part_4_4_0"}}, + }, + RebalancedFiles: nil, + Database: "default", + Table: "test", + Query: "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id", + Size: nil, + TotalBytes: 1000, + Mutations: nil, + MetadataOnly: false, + LocalFile: "/dev/null", + } + tableMetadataAfterDownload := []metadata.TableMetadata{invalidTable} + // hdd2 not exists diskType + delete(invalidRemoteBackup.DiskTypes, "hdd2") + tableMetadataAfterDownloadRepacked := make([]*metadata.TableMetadata, len(tableMetadataAfterDownload)) + for i := range tableMetadataAfterDownload { + tableMetadataAfterDownloadRepacked[i] = &tableMetadataAfterDownload[i] + } + err := b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownloadRepacked, baseDisks, invalidRemoteBackup) + assert.Error(t, err) + assert.Equal(t, + "disk: hdd2 not found in disk_types section map[string]string{\"default\":\"local\", \"s3\":\"s3\", \"s3_disk2\":\"s3\"} in Test/metadata.json", + err.Error(), + ) + + // invalid disk_type + invalidRemoteBackup.DiskTypes["hdd2"] = "unknown" + tableMetadataAfterDownloadRepacked = make([]*metadata.TableMetadata, len(tableMetadataAfterDownload)) + for i := range tableMetadataAfterDownload { + tableMetadataAfterDownloadRepacked[i] = &tableMetadataAfterDownload[i] + } + err = b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownloadRepacked, baseDisks, invalidRemoteBackup) + assert.Error(t, err) + assert.Equal(t, "disk: hdd2, diskType: unknown not found in system.disks", err.Error()) + + // invalid storage_policy + invalidRemoteBackup.DiskTypes["hdd2"] = "local" + invalidTable.Parts = map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "hdd2": {{Name: "part_3_3_0"}, {Name: "part_4_4_0"}}, + "s3": {{Name: "part_5_5_0"}, {Name: "part_6_6_0"}}, + "s3_disk2": {{Name: "part_7_7_0"}, {Name: "part_8_8_0"}}, + } + + invalidTable.Table = "test3" + invalidTable.Query = "CREATE TABLE default.test3(id UInt64) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='invalid'" + tableMetadataAfterDownloadRepacked = []*metadata.TableMetadata{&invalidTable} + err = b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownloadRepacked, baseDisks, invalidRemoteBackup) + assert.Error(t, err) + matched, matchErr := regexp.MatchString(`storagePolicy: invalid with diskType: \w+ not found in system.disks`, err.Error()) + assert.NoError(t, matchErr) + assert.True(t, matched) + + // no free space + invalidDisks := baseDisks + invalidDisks[0].FreeSpace = 0 + invalidTable.Table = "test4" + invalidTable.Query = "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id SETTINGS storage_policy='default'" + invalidTable.Parts = map[string][]metadata.Part{ + "default": {{Name: "part_1_1_0"}, {Name: "part_2_2_0"}}, + "hdd2": {{Name: "part_3_3_0"}, {Name: "part_4_4_0"}}, + } + tableMetadataAfterDownloadRepacked = []*metadata.TableMetadata{&invalidTable} + err = b.reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownloadRepacked, invalidDisks, invalidRemoteBackup) + assert.Error(t, err) + assert.Equal(t, "250B free space, not found in system.disks with `local` type", err.Error()) + +} diff --git a/pkg/backup/list.go b/pkg/backup/list.go index ce5ea97f..f1939a2f 100644 --- a/pkg/backup/list.go +++ b/pkg/backup/list.go @@ -4,19 +4,22 @@ import ( "context" "encoding/json" "fmt" + "github.com/ricochet2200/go-disk-usage/du" "io" "os" "path" + "path/filepath" "sort" "strings" "text/tabwriter" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/custom" - "github.com/Altinity/clickhouse-backup/pkg/metadata" - "github.com/Altinity/clickhouse-backup/pkg/status" - "github.com/Altinity/clickhouse-backup/pkg/storage" - "github.com/Altinity/clickhouse-backup/pkg/utils" + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/custom" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "github.com/Altinity/clickhouse-backup/v2/pkg/status" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" + "github.com/rs/zerolog/log" ) @@ -43,7 +46,7 @@ func printBackupsRemote(w io.Writer, backupList []storage.Backup, format string) fmt.Println(backupList[len(backupList)-1].BackupName) case "penult", "prev", "previous", "p": if len(backupList) < 2 { - return fmt.Errorf("no penult backup is found") + return fmt.Errorf("no previous backup is found") } fmt.Println(backupList[len(backupList)-2].BackupName) case "all", "": @@ -51,15 +54,9 @@ func printBackupsRemote(w io.Writer, backupList []storage.Backup, format string) // fmt.Println("no backups found") // } for _, backup := range backupList { - size := utils.FormatBytes(backup.DataSize + backup.MetadataSize) - if backup.CompressedSize > 0 { - size = utils.FormatBytes(backup.CompressedSize + backup.MetadataSize) - } + size := utils.FormatBytes(backup.GetFullSize()) description := backup.DataFormat uploadDate := backup.UploadDate.Format("02/01/2006 15:04:05") - if backup.Legacy { - description += ", old-format" - } if backup.Tags != "" { description += ", " + backup.Tags } @@ -94,23 +91,20 @@ func printBackupsLocal(ctx context.Context, w io.Writer, backupList []LocalBacku } fmt.Println(backupList[len(backupList)-2].BackupName) case "all", "": - // if len(backupList) == 0 { - // fmt.Println("no backups found") - // } for _, backup := range backupList { select { case <-ctx.Done(): return ctx.Err() default: - size := utils.FormatBytes(backup.DataSize + backup.MetadataSize) - if backup.CompressedSize > 0 { - size = utils.FormatBytes(backup.CompressedSize + backup.MetadataSize) - } + size := utils.FormatBytes(backup.GetFullSize()) description := backup.DataFormat - creationDate := backup.CreationDate.Format("02/01/2006 15:04:05") - if backup.Legacy { - size = "???" + if backup.Tags != "" { + if description != "" { + description += ", " + } + description += backup.Tags } + creationDate := backup.CreationDate.Format("02/01/2006 15:04:05") required := "" if backup.RequiredBackup != "" { required = "+" + backup.RequiredBackup @@ -168,26 +162,43 @@ func (b *Backuper) GetLocalBackups(ctx context.Context, disks []clickhouse.Disk) } if disks == nil { disks = []clickhouse.Disk{ - {Name: "default", Path: "/var/lib/clickhouse"}, + { + Name: "default", + Path: "/var/lib/clickhouse", + Type: "local", + FreeSpace: du.NewDiskUsage("/var/lib/clickhouse").Free(), + StoragePolicies: []string{"default"}, + }, } } - defaultDataPath, err := b.ch.GetDefaultPath(disks) - if err != nil { - return nil, nil, err - } var result []LocalBackup - allBackupPaths := []string{path.Join(defaultDataPath, "backup")} - if b.cfg.ClickHouse.UseEmbeddedBackupRestore { - for _, disk := range disks { - select { - case <-ctx.Done(): - return nil, nil, ctx.Err() - default: - if disk.IsBackup || disk.Name == b.cfg.ClickHouse.EmbeddedBackupDisk { - allBackupPaths = append(allBackupPaths, disk.Path) - } + allBackupPaths := []string{} + for _, disk := range disks { + if disk.IsBackup || disk.Name == b.cfg.ClickHouse.EmbeddedBackupDisk { + allBackupPaths = append(allBackupPaths, disk.Path) + } else { + allBackupPaths = append(allBackupPaths, path.Join(disk.Path, "backup")) + } + } + addBrokenBackupIfNotExists := func(result []LocalBackup, name string, info os.FileInfo, broken string) []LocalBackup { + backupAlreadyExists := false + for _, backup := range result { + if backup.BackupName == name { + backupAlreadyExists = true + break } } + // add broken backup if not exists + if !backupAlreadyExists { + result = append(result, LocalBackup{ + BackupMetadata: metadata.BackupMetadata{ + BackupName: name, + CreationDate: info.ModTime(), + }, + Broken: broken, + }) + } + return result } l := len(allBackupPaths) for i, backupPath := range allBackupPaths { @@ -219,28 +230,36 @@ func (b *Backuper) GetLocalBackups(ctx context.Context, disks []clickhouse.Disk) } backupMetafilePath := path.Join(backupPath, name, "metadata.json") backupMetadataBody, err := os.ReadFile(backupMetafilePath) - if os.IsNotExist(err) { - // Legacy backup - result = append(result, LocalBackup{ - BackupMetadata: metadata.BackupMetadata{ - BackupName: name, - CreationDate: info.ModTime(), - }, - Legacy: true, - }) + if err != nil { + if !os.IsNotExist(err) { + log.Warn().Msgf("list can't read %s error: %s", backupMetafilePath, err) + } + result = addBrokenBackupIfNotExists(result, name, info, "broken metadata.json not found") continue } var backupMetadata metadata.BackupMetadata - if err := json.Unmarshal(backupMetadataBody, &backupMetadata); err != nil { - return nil, disks, err + if parseErr := json.Unmarshal(backupMetadataBody, &backupMetadata); parseErr != nil { + result = addBrokenBackupIfNotExists(result, name, info, fmt.Sprintf("parse metadata.json error: %v", parseErr)) + continue } - result = append(result, LocalBackup{ - BackupMetadata: backupMetadata, - Legacy: false, - }) + brokenBackupIsAlreadyExists := false + for i, backup := range result { + if backup.BackupName == backupMetadata.BackupName { + brokenBackupIsAlreadyExists = true + result[i].BackupMetadata = backupMetadata + result[i].Broken = "" + break + } + } + if !brokenBackupIsAlreadyExists { + result = append(result, LocalBackup{ + BackupMetadata: backupMetadata, + }) + } + } if closeErr := d.Close(); closeErr != nil { - log.Error().Msgf("can't close %s openError: %v", backupPath, closeErr) + log.Error().Msgf("can't close %s error: %v", backupPath, closeErr) } } } @@ -382,13 +401,33 @@ func (b *Backuper) GetTables(ctx context.Context, tablePattern string) ([]clickh } // PrintTables - print all tables suitable for backup -func (b *Backuper) PrintTables(printAll bool, tablePattern string) error { +func (b *Backuper) PrintTables(printAll bool, tablePattern, remoteBackup string) error { + var err error ctx, cancel, _ := status.Current.GetContextWithCancel(status.NotFromAPI) defer cancel() - if err := b.ch.Connect(); err != nil { + if err = b.ch.Connect(); err != nil { return fmt.Errorf("can't connect to clickhouse: %v", err) } defer b.ch.Close() + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', tabwriter.DiscardEmptyColumns) + if remoteBackup == "" { + if err = b.printTablesLocal(ctx, tablePattern, printAll, w); err != nil { + return err + } + } else { + if err = b.printTablesRemote(ctx, remoteBackup, tablePattern, printAll, w); err != nil { + return err + } + + } + if err := w.Flush(); err != nil { + log.Error().Msgf("can't flush tabular writer error: %v", err) + } + return nil +} + +func (b *Backuper) printTablesLocal(ctx context.Context, tablePattern string, printAll bool, w *tabwriter.Writer) error { + logger := log.With().Str("logger", "PrintTablesLocal").Logger() allTables, err := b.GetTables(ctx, tablePattern) if err != nil { return err @@ -397,7 +436,6 @@ func (b *Backuper) PrintTables(printAll bool, tablePattern string) error { if err != nil { return err } - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', tabwriter.DiscardEmptyColumns) for _, table := range allTables { if table.Skip && !printAll { continue @@ -408,16 +446,95 @@ func (b *Backuper) PrintTables(printAll bool, tablePattern string) error { } if table.Skip { if bytes, err := fmt.Fprintf(w, "%s.%s\t%s\t%v\tskip\n", table.Database, table.Name, utils.FormatBytes(table.TotalBytes), strings.Join(tableDisks, ",")); err != nil { - log.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err) + logger.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err) } continue } if bytes, err := fmt.Fprintf(w, "%s.%s\t%s\t%v\t%v\n", table.Database, table.Name, utils.FormatBytes(table.TotalBytes), strings.Join(tableDisks, ","), table.BackupType); err != nil { - log.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err) + logger.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err) } } - if err := w.Flush(); err != nil { - log.Error().Msgf("can't flush tabular writer error: %v", err) + return nil +} + +func (b *Backuper) GetTablesRemote(ctx context.Context, backupName string, tablePattern string) ([]clickhouse.Table, error) { + if !b.ch.IsOpen { + if err := b.ch.Connect(); err != nil { + return []clickhouse.Table{}, fmt.Errorf("can't connect to clickhouse: %v", err) + } + defer b.ch.Close() + } + if b.cfg.General.RemoteStorage == "none" || b.cfg.General.RemoteStorage == "custom" { + return nil, fmt.Errorf("GetTablesRemote does not support `none` and `custom` remote storage") + } + if b.dst == nil { + bd, err := storage.NewBackupDestination(ctx, b.cfg, b.ch, false, "") + if err != nil { + return nil, err + } + err = bd.Connect(ctx) + if err != nil { + return nil, fmt.Errorf("can't connect to remote storage: %v", err) + } + defer func() { + if err := bd.Close(ctx); err != nil { + log.Warn().Msgf("can't close BackupDestination error: %v", err) + } + }() + + b.dst = bd + } + backupList, err := b.dst.BackupList(ctx, true, backupName) + if err != nil { + return nil, err } + + var tables []clickhouse.Table + tablePatterns := []string{"*"} + + if tablePattern != "" { + tablePatterns = strings.Split(tablePattern, ",") + } + + for _, remoteBackup := range backupList { + if remoteBackup.BackupName == backupName { + for _, t := range remoteBackup.Tables { + isInformationSchema := IsInformationSchema(t.Database) + tableName := fmt.Sprintf("%s.%s", t.Database, t.Table) + shallSkipped := b.shouldSkipByTableName(tableName) + matched := false + for _, p := range tablePatterns { + if matched, _ = filepath.Match(strings.Trim(p, " \t\r\n"), tableName); matched { + break + } + } + tables = append(tables, clickhouse.Table{ + Database: t.Database, + Name: t.Table, + Skip: !matched || (isInformationSchema || shallSkipped), + }) + } + } + } + + return tables, nil +} + +// printTablesRemote https://github.com/Altinity/clickhouse-backup/issues/778 +func (b *Backuper) printTablesRemote(ctx context.Context, backupName string, tablePattern string, printAll bool, w *tabwriter.Writer) error { + tables, err := b.GetTablesRemote(ctx, backupName, tablePattern) + if err != nil { + return err + } + + for _, t := range tables { + if t.Skip && !printAll { + continue + } + if bytes, err := fmt.Fprintf(w, "%s.%s\tskip=%v\n", t.Database, t.Name, t.Skip); err != nil { + log.Error().Msgf("fmt.Fprintf write %d bytes return error: %v", bytes, err) + } + } + return nil } diff --git a/pkg/backup/restore.go b/pkg/backup/restore.go index d0e5d12d..d075b6c1 100644 --- a/pkg/backup/restore.go +++ b/pkg/backup/restore.go @@ -1,14 +1,13 @@ package backup import ( + "bufio" "context" "encoding/json" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/keeper" - "github.com/Altinity/clickhouse-backup/pkg/status" - "github.com/Altinity/clickhouse-backup/pkg/storage" - "github.com/Altinity/clickhouse-backup/pkg/storage/object_disk" + apexLog "github.com/apex/log" + "github.com/rs/zerolog" + "io" "io/fs" "net/url" "os" @@ -17,35 +16,49 @@ import ( "path/filepath" "regexp" "strings" + "sync/atomic" "time" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/common" - "github.com/Altinity/clickhouse-backup/pkg/filesystemhelper" - "github.com/Altinity/clickhouse-backup/pkg/metadata" - "github.com/Altinity/clickhouse-backup/pkg/utils" "github.com/mattn/go-shellwords" recursiveCopy "github.com/otiai10/copy" "github.com/rs/zerolog/log" "github.com/yargevad/filepathx" + "golang.org/x/sync/errgroup" + "golang.org/x/text/cases" + "golang.org/x/text/language" + + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/common" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/filesystemhelper" + "github.com/Altinity/clickhouse-backup/v2/pkg/keeper" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "github.com/Altinity/clickhouse-backup/v2/pkg/status" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage/object_disk" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" ) var CreateDatabaseRE = regexp.MustCompile(`(?m)^CREATE DATABASE (\s*)(\S+)(\s*)`) // Restore - restore tables matched by tablePattern from backupName -func (b *Backuper) Restore(backupName, tablePattern string, databaseMapping, partitions []string, schemaOnly, dataOnly, dropTable, ignoreDependencies, restoreRBAC, rbacOnly, restoreConfigs, configsOnly bool, commandId int) error { +func (b *Backuper) Restore(backupName, tablePattern string, databaseMapping, tableMapping, partitions []string, schemaOnly, dataOnly, dropExists, ignoreDependencies, restoreRBAC, rbacOnly, restoreConfigs, configsOnly bool, backupVersion string, commandId int) error { ctx, cancel, err := status.Current.GetContextWithCancel(commandId) if err != nil { return err } ctx, cancel = context.WithCancel(ctx) defer cancel() + startRestore := time.Now() backupName = utils.CleanBackupNameRE.ReplaceAllString(backupName, "") - if err := b.prepareRestoreDatabaseMapping(databaseMapping); err != nil { + if err := b.prepareRestoreMapping(databaseMapping, "database"); err != nil { + return err + } + if err := b.prepareRestoreMapping(tableMapping, "table"); err != nil { return err } - doRestoreData := !schemaOnly || dataOnly + doRestoreData := (!schemaOnly && !rbacOnly && !configsOnly) || dataOnly if err := b.ch.Connect(); err != nil { return fmt.Errorf("can't connect to clickhouse: %v", err) @@ -60,75 +73,81 @@ func (b *Backuper) Restore(backupName, tablePattern string, databaseMapping, par if err != nil { return err } - defaultDataPath, err := b.ch.GetDefaultPath(disks) + version, err := b.ch.GetVersion(ctx) + if err != nil { + return err + } + b.DefaultDataPath, err = b.ch.GetDefaultPath(disks) if err != nil { log.Warn().Msgf("%v", err) return ErrUnknownClickhouseDataPath } - backupMetafileLocalPaths := []string{path.Join(defaultDataPath, "backup", backupName, "metadata.json")} + if b.cfg.General.RestoreSchemaOnCluster != "" { + if b.cfg.General.RestoreSchemaOnCluster, err = b.ch.ApplyMacros(ctx, b.cfg.General.RestoreSchemaOnCluster); err != nil { + log.Warn().Msgf("%v", err) + return err + } + } + backupMetafileLocalPaths := []string{path.Join(b.DefaultDataPath, "backup", backupName, "metadata.json")} var backupMetadataBody []byte - embeddedBackupPath, err := b.ch.GetEmbeddedBackupPath(disks) - if err == nil && embeddedBackupPath != "" { - backupMetafileLocalPaths = append(backupMetafileLocalPaths, path.Join(embeddedBackupPath, backupName, "metadata.json")) + b.EmbeddedBackupDataPath, err = b.ch.GetEmbeddedBackupPath(disks) + if err == nil && b.EmbeddedBackupDataPath != "" { + backupMetafileLocalPaths = append(backupMetafileLocalPaths, path.Join(b.EmbeddedBackupDataPath, backupName, "metadata.json")) } else if b.cfg.ClickHouse.UseEmbeddedBackupRestore && b.cfg.ClickHouse.EmbeddedBackupDisk == "" { - log.Warn().Msgf("%v", err) + b.EmbeddedBackupDataPath = b.DefaultDataPath } else if err != nil { return err } for _, metadataPath := range backupMetafileLocalPaths { backupMetadataBody, err = os.ReadFile(metadataPath) - if err == nil && embeddedBackupPath != "" { - b.isEmbedded = strings.HasPrefix(metadataPath, embeddedBackupPath) + if err == nil { break } } - if b.cfg.General.RestoreSchemaOnCluster != "" { - b.cfg.General.RestoreSchemaOnCluster, err = b.ch.ApplyMacros(ctx, b.cfg.General.RestoreSchemaOnCluster) + if err != nil { + return err } - if err == nil { - backupMetadata := metadata.BackupMetadata{} - if err := json.Unmarshal(backupMetadataBody, &backupMetadata); err != nil { - return err - } + backupMetadata := metadata.BackupMetadata{} + if err := json.Unmarshal(backupMetadataBody, &backupMetadata); err != nil { + return err + } + b.isEmbedded = strings.Contains(backupMetadata.Tags, "embedded") - if schemaOnly || doRestoreData { - for _, database := range backupMetadata.Databases { - targetDB := database.Name - if !IsInformationSchema(targetDB) { - if err = b.restoreEmptyDatabase(ctx, targetDB, tablePattern, database, dropTable, schemaOnly, ignoreDependencies); err != nil { - return err - } - } - } - } - // do not create UDF when use --data flag, https://github.com/Altinity/clickhouse-backup/issues/697 - if schemaOnly || (schemaOnly == dataOnly) { - for _, function := range backupMetadata.Functions { - if err = b.ch.CreateUserDefinedFunction(function.Name, function.CreateQuery, b.cfg.General.RestoreSchemaOnCluster); err != nil { + if schemaOnly || doRestoreData { + for _, database := range backupMetadata.Databases { + targetDB := database.Name + if !IsInformationSchema(targetDB) { + if err = b.restoreEmptyDatabase(ctx, targetDB, tablePattern, database, dropExists, schemaOnly, ignoreDependencies, version); err != nil { return err } } } - if len(backupMetadata.Tables) == 0 { - log.Warn().Msgf("'%s' doesn't contains tables for restore", backupName) - if (!restoreRBAC) && (!restoreConfigs) { - return nil + } + if len(backupMetadata.Tables) == 0 { + // corner cases for https://github.com/Altinity/clickhouse-backup/issues/832 + if !restoreRBAC && !rbacOnly && !restoreConfigs && !configsOnly { + if !b.cfg.General.AllowEmptyBackups { + err = fmt.Errorf("'%s' doesn't contains tables for restore, if you need it, you can setup `allow_empty_backups: true` in `general` config section", backupName) + log.Error().Msgf("%v", err) + return err } + log.Warn().Msgf("'%s' doesn't contains tables for restore", backupName) + return nil } - } else if !os.IsNotExist(err) { // Legacy backups don't contain metadata.json - return err } needRestart := false - if (rbacOnly || restoreRBAC) && !b.isEmbedded { - if err := b.restoreRBAC(ctx, backupName, disks); err != nil { + if rbacOnly || restoreRBAC { + if err := b.restoreRBAC(ctx, backupName, disks, version, dropExists); err != nil { return err } + log.Info().Msgf("RBAC successfully restored") needRestart = true } - if (configsOnly || restoreConfigs) && !b.isEmbedded { + if configsOnly || restoreConfigs { if err := b.restoreConfigs(backupName, disks); err != nil { return err } + log.Info().Msgf("CONFIGS successfully restored") needRestart = true } @@ -140,21 +159,127 @@ func (b *Backuper) Restore(backupName, tablePattern string, databaseMapping, par return nil } } + isObjectDiskPresents := false + if b.cfg.General.RemoteStorage != "custom" { + for _, d := range disks { + if isObjectDiskPresents = b.isDiskTypeObject(d.Type); isObjectDiskPresents { + break + } + } + } + if (b.cfg.ClickHouse.UseEmbeddedBackupRestore && b.cfg.ClickHouse.EmbeddedBackupDisk == "") || isObjectDiskPresents { + if b.dst, err = storage.NewBackupDestination(ctx, b.cfg, b.ch, false, backupName); err != nil { + return err + } + if err = b.dst.Connect(ctx); err != nil { + return fmt.Errorf("BackupDestination for embedded or object disk: can't connect to %s: %v", b.dst.Kind(), err) + } + defer func() { + if err := b.dst.Close(ctx); err != nil { + log.Warn().Msgf("can't close BackupDestination error: %v", err) + } + }() + } + var tablesForRestore ListOfTables + var partitionsNames map[metadata.TableTitle][]string + if tablePattern == "" { + tablePattern = "*" + } + metadataPath := path.Join(b.DefaultDataPath, "backup", backupName, "metadata") + if b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk != "" { + metadataPath = path.Join(b.EmbeddedBackupDataPath, backupName, "metadata") + } - if schemaOnly || (schemaOnly == dataOnly) { - if err := b.RestoreSchema(ctx, backupName, tablePattern, dropTable, ignoreDependencies, disks); err != nil { + if !rbacOnly && !configsOnly { + tablesForRestore, partitionsNames, err = b.getTablesForRestoreLocal(ctx, backupName, metadataPath, tablePattern, dropExists, partitions) + if err != nil { return err } } - if dataOnly || (schemaOnly == dataOnly) { - if err := b.RestoreData(ctx, backupName, tablePattern, partitions, disks); err != nil { + if schemaOnly || dropExists || (schemaOnly == dataOnly && !rbacOnly && !configsOnly) { + if err = b.RestoreSchema(ctx, backupName, backupMetadata, disks, tablesForRestore, ignoreDependencies, version); err != nil { return err } } - log.Info().Msg("done") + // https://github.com/Altinity/clickhouse-backup/issues/756 + if dataOnly && !schemaOnly && !rbacOnly && !configsOnly && len(partitions) > 0 { + if err = b.dropExistPartitions(ctx, tablesForRestore, partitionsNames, partitions, version); err != nil { + return err + } + + } + if dataOnly || (schemaOnly == dataOnly && !rbacOnly && !configsOnly) { + if err := b.RestoreData(ctx, backupName, backupMetadata, dataOnly, metadataPath, tablePattern, partitions, disks, version); err != nil { + return err + } + } + // do not create UDF when use --data, --rbac-only, --configs-only flags, https://github.com/Altinity/clickhouse-backup/issues/697 + if schemaOnly || (schemaOnly == dataOnly && !rbacOnly && !configsOnly) { + for _, function := range backupMetadata.Functions { + if err = b.ch.CreateUserDefinedFunction(function.Name, function.CreateQuery, b.cfg.General.RestoreSchemaOnCluster); err != nil { + return err + } + } + } + + //clean partially downloaded requiredBackup + if backupMetadata.RequiredBackup != "" { + if err = b.cleanPartialRequiredBackup(ctx, disks, backupMetadata.BackupName); err != nil { + return err + } + } + + log.Info().Fields(map[string]interface{}{ + "duration": utils.HumanizeDuration(time.Since(startRestore)), + "version": backupVersion, + }).Msg("done") return nil } +func (b *Backuper) getTablesForRestoreLocal(ctx context.Context, backupName string, metadataPath string, tablePattern string, dropTable bool, partitions []string) (ListOfTables, map[metadata.TableTitle][]string, error) { + var tablesForRestore ListOfTables + var partitionsNames map[metadata.TableTitle][]string + info, err := os.Stat(metadataPath) + // corner cases for https://github.com/Altinity/clickhouse-backup/issues/832 + if err != nil { + if !b.cfg.General.AllowEmptyBackups { + return nil, nil, err + } + if !os.IsNotExist(err) { + return nil, nil, err + } + return nil, nil, nil + } + if !info.IsDir() { + return nil, nil, fmt.Errorf("%s is not a dir", metadataPath) + } + tablesForRestore, partitionsNames, err = b.getTableListByPatternLocal(ctx, metadataPath, tablePattern, dropTable, partitions) + if err != nil { + return nil, nil, err + } + // if restore-database-mapping is specified, create database in mapping rules instead of in backup files. + if len(b.cfg.General.RestoreDatabaseMapping) > 0 { + err = changeTableQueryToAdjustDatabaseMapping(&tablesForRestore, b.cfg.General.RestoreDatabaseMapping) + if err != nil { + return nil, nil, err + } + } + + // if restore-table-mapping is specified, create table in mapping rules instead of in backup files. + // https://github.com/Altinity/clickhouse-backup/issues/937 + if len(b.cfg.General.RestoreTableMapping) > 0 { + err = changeTableQueryToAdjustTableMapping(&tablesForRestore, b.cfg.General.RestoreTableMapping) + if err != nil { + return nil, nil, err + } + } + + if len(tablesForRestore) == 0 { + return nil, nil, fmt.Errorf("not found schemas by %s in %s, also check skip_tables and skip_table_engines setting", tablePattern, backupName) + } + return tablesForRestore, partitionsNames, nil +} + func (b *Backuper) restartClickHouse(ctx context.Context, backupName string) error { log.Warn().Msgf("%s contains `access` or `configs` directory, so we need exec %s", backupName, b.ch.Config.RestartCommand) for _, cmd := range strings.Split(b.ch.Config.RestartCommand, ";") { @@ -206,23 +331,24 @@ func (b *Backuper) executeShellCommandWithTimeout(ctx context.Context, cmd strin } else { out, err = exec.CommandContext(shellCtx, shellCmd[0]).CombinedOutput() } + log.Debug().Msgf(string(out)) if err != nil { - log.Debug().Msg(string(out)) log.Warn().Msgf("restart exec: %s, error: %v", cmd, err) } return nil } -func (b *Backuper) restoreEmptyDatabase(ctx context.Context, targetDB, tablePattern string, database metadata.DatabasesMeta, dropTable, schemaOnly, ignoreDependencies bool) error { +func (b *Backuper) restoreEmptyDatabase(ctx context.Context, targetDB, tablePattern string, database metadata.DatabasesMeta, dropTable, schemaOnly, ignoreDependencies bool, version int) error { isMapped := false if targetDB, isMapped = b.cfg.General.RestoreDatabaseMapping[database.Name]; !isMapped { targetDB = database.Name } // https://github.com/Altinity/clickhouse-backup/issues/583 + // https://github.com/Altinity/clickhouse-backup/issues/663 if ShallSkipDatabase(b.cfg, targetDB, tablePattern) { return nil } - //https://github.com/Altinity/clickhouse-backup/issues/514 + // https://github.com/Altinity/clickhouse-backup/issues/514 if schemaOnly && dropTable { onCluster := "" if b.cfg.General.RestoreSchemaOnCluster != "" { @@ -231,14 +357,13 @@ func (b *Backuper) restoreEmptyDatabase(ctx context.Context, targetDB, tablePatt // https://github.com/Altinity/clickhouse-backup/issues/651 settings := "" if ignoreDependencies { - version, err := b.ch.GetVersion(ctx) - if err != nil { - return err - } if version >= 21012000 { settings = "SETTINGS check_table_dependencies=0" } } + if _, err := os.Create(path.Join(b.DefaultDataPath, "/flags/force_drop_table")); err != nil { + return err + } if err := b.ch.QueryContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s` %s SYNC %s", targetDB, onCluster, settings)); err != nil { return err } @@ -251,26 +376,49 @@ func (b *Backuper) restoreEmptyDatabase(ctx context.Context, targetDB, tablePatt return nil } -func (b *Backuper) prepareRestoreDatabaseMapping(databaseMapping []string) error { - for i := 0; i < len(databaseMapping); i++ { - splitByCommas := strings.Split(databaseMapping[i], ",") +func (b *Backuper) prepareRestoreMapping(objectMapping []string, objectType string) error { + if objectType != "database" && objectType != "table" { + return fmt.Errorf("objectType must be one of `database` or `table`") + } + for i := 0; i < len(objectMapping); i++ { + splitByCommas := strings.Split(objectMapping[i], ",") for _, m := range splitByCommas { splitByColon := strings.Split(m, ":") if len(splitByColon) != 2 { - return fmt.Errorf("restore-database-mapping %s should only have srcDatabase:destinationDatabase format for each map rule", m) + objectTypeTitleCase := cases.Title(language.Und).String(objectType) + return fmt.Errorf("restore-%s-mapping %s should only have src%s:destination%s format for each map rule", objectType, m, objectTypeTitleCase, objectTypeTitleCase) + } + if objectType == "database" { + b.cfg.General.RestoreDatabaseMapping[splitByColon[0]] = splitByColon[1] + } else { + b.cfg.General.RestoreTableMapping[splitByColon[0]] = splitByColon[1] } - b.cfg.General.RestoreDatabaseMapping[splitByColon[0]] = splitByColon[1] } } return nil } // restoreRBAC - copy backup_name>/rbac folder to access_data_path -func (b *Backuper) restoreRBAC(ctx context.Context, backupName string, disks []clickhouse.Disk) error { +func (b *Backuper) restoreRBAC(ctx context.Context, backupName string, disks []clickhouse.Disk, version int, dropExists bool) error { accessPath, err := b.ch.GetAccessManagementPath(ctx, nil) if err != nil { return err } + var k *keeper.Keeper + replicatedUserDirectories := make([]clickhouse.UserDirectory, 0) + if err = b.ch.SelectContext(ctx, &replicatedUserDirectories, "SELECT name FROM system.user_directories WHERE type='replicated'"); err == nil && len(replicatedUserDirectories) > 0 { + k = &keeper.Keeper{} + if connErr := k.Connect(ctx, b.ch); connErr != nil { + return fmt.Errorf("but can't connect to keeper: %v", connErr) + } + defer k.Close() + } + + // https://github.com/Altinity/clickhouse-backup/issues/851 + if err = b.restoreRBACResolveAllConflicts(ctx, backupName, accessPath, version, k, replicatedUserDirectories, dropExists); err != nil { + return err + } + if err = b.restoreBackupRelatedDir(backupName, "access", accessPath, disks, []string{"*.jsonl"}); err == nil { markFile := path.Join(accessPath, "need_rebuild_lists.mark") log.Info().Msgf("create %s for properly rebuild RBAC after restart clickhouse-server", markFile) @@ -295,62 +443,324 @@ func (b *Backuper) restoreRBAC(ctx context.Context, backupName string, disks []c if err != nil && !os.IsNotExist(err) { return err } - if err = b.restoreRBACReplicated(ctx, backupName, "access", disks); err != nil { + if err != nil && os.IsNotExist(err) { + return nil + } + if err = b.restoreRBACReplicated(backupName, "access", k, replicatedUserDirectories); err != nil && !os.IsNotExist(err) { return err } return nil } -func (b *Backuper) restoreRBACReplicated(ctx context.Context, backupName string, backupPrefixDir string, disks []clickhouse.Disk) error { - defaultDataPath, err := b.ch.GetDefaultPath(disks) +func (b *Backuper) restoreRBACResolveAllConflicts(ctx context.Context, backupName string, accessPath string, version int, k *keeper.Keeper, replicatedUserDirectories []clickhouse.UserDirectory, dropExists bool) error { + backupAccessPath := path.Join(b.DefaultDataPath, "backup", backupName, "access") + + walkErr := filepath.Walk(backupAccessPath, func(fPath string, fInfo fs.FileInfo, err error) error { + if err != nil { + return err + } + if fInfo.IsDir() { + return nil + } + if strings.HasSuffix(fPath, ".sql") { + sql, readErr := os.ReadFile(fPath) + if readErr != nil { + return readErr + } + if resolveErr := b.resolveRBACConflictIfExist(ctx, string(sql), accessPath, version, k, replicatedUserDirectories, dropExists); resolveErr != nil { + return resolveErr + } + log.Debug().Msgf("%s b.resolveRBACConflictIfExist(%s) no error", fPath, string(sql)) + } + if strings.HasSuffix(fPath, ".jsonl") { + file, openErr := os.Open(fPath) + if openErr != nil { + return openErr + } + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + data := keeper.DumpNode{} + jsonErr := json.Unmarshal([]byte(line), &data) + if jsonErr != nil { + log.Error().Msgf("can't %s json.Unmarshal error: %v line: %s", fPath, line, jsonErr) + continue + } + if strings.HasPrefix(data.Path, "uuid/") { + if resolveErr := b.resolveRBACConflictIfExist(ctx, data.Value, accessPath, version, k, replicatedUserDirectories, dropExists); resolveErr != nil { + return resolveErr + } + log.Debug().Msgf("%s:%s b.resolveRBACConflictIfExist(%s) no error", fPath, data.Path, data.Value) + } + + } + if scanErr := scanner.Err(); scanErr != nil { + return scanErr + } + + if closeErr := file.Close(); closeErr != nil { + log.Warn().Msgf("can't close %s error: %v", fPath, closeErr) + } + + } + return nil + }) + if !os.IsNotExist(walkErr) { + return walkErr + } + return nil +} + +func (b *Backuper) resolveRBACConflictIfExist(ctx context.Context, sql string, accessPath string, version int, k *keeper.Keeper, replicatedUserDirectories []clickhouse.UserDirectory, dropExists bool) error { + kind, name, detectErr := b.detectRBACObject(sql) + if detectErr != nil { + return detectErr + } + if isExists, existsRBACType, existsRBACObjectId := b.isRBACExists(ctx, kind, name, accessPath, version, k, replicatedUserDirectories); isExists { + log.Warn().Msgf("RBAC object kind=%s, name=%s already present, will %s", kind, name, b.cfg.General.RBACConflictResolution) + if b.cfg.General.RBACConflictResolution == "recreate" || dropExists { + if dropErr := b.dropExistsRBAC(ctx, kind, name, accessPath, existsRBACType, existsRBACObjectId, k); dropErr != nil { + return dropErr + } + return nil + } + if b.cfg.General.RBACConflictResolution == "fail" { + return fmt.Errorf("RBAC object kind=%s, name=%s already present, change ", kind, name) + } + } + return nil +} + +func (b *Backuper) isRBACExists(ctx context.Context, kind string, name string, accessPath string, version int, k *keeper.Keeper, replicatedUserDirectories []clickhouse.UserDirectory) (bool, string, string) { + //search in sql system.users, system.quotas, system.row_policies, system.roles, system.settings_profiles + if version > 22003000 { + var rbacSystemTableNames = map[string]string{ + "ROLE": "roles", + "ROW POLICY": "row_policies", + "SETTINGS PROFILE": "settings_profiles", + "QUOTA": "quotas", + "USER": "users", + } + systemTable, systemTableExists := rbacSystemTableNames[kind] + if !systemTableExists { + log.Error().Msgf("unsupported RBAC object kind: %s", kind) + return false, "", "" + } + isRBACExistsSQL := fmt.Sprintf("SELECT toString(id) AS id, name FROM `system`.`%s` WHERE name=? LIMIT 1", systemTable) + existsRBACRow := make([]clickhouse.RBACObject, 0) + if err := b.ch.SelectContext(ctx, &existsRBACRow, isRBACExistsSQL, name); err != nil { + log.Warn().Msgf("RBAC object resolve failed, check SQL GRANTS or settings for user which you use to connect to clickhouse-server, kind: %s, name: %s, error: %v", kind, name, err) + return false, "", "" + } + if len(existsRBACRow) == 0 { + return false, "", "" + } + return true, "sql", existsRBACRow[0].Id + } + + checkRBACExists := func(sql string) bool { + existsKind, existsName, detectErr := b.detectRBACObject(sql) + if detectErr != nil { + log.Warn().Msgf("isRBACExists error: %v", detectErr) + return false + } + if existsKind == kind && existsName == name { + return true + } + return false + } + + // search in local user directory + if sqlFiles, globErr := filepath.Glob(path.Join(accessPath, "*.sql")); globErr == nil { + for _, f := range sqlFiles { + sql, readErr := os.ReadFile(f) + if readErr != nil { + log.Warn().Msgf("read %s error: %v", f, readErr) + continue + } + if checkRBACExists(string(sql)) { + return true, "local", strings.TrimSuffix(filepath.Base(f), filepath.Ext(f)) + } + } + } else { + log.Warn().Msgf("access/*.sql error: %v", globErr) + } + + //search in keeper replicated user directory + if k != nil && len(replicatedUserDirectories) > 0 { + for _, userDirectory := range replicatedUserDirectories { + replicatedAccessPath, getAccessErr := k.GetReplicatedAccessPath(userDirectory.Name) + if getAccessErr != nil { + log.Warn().Msgf("b.isRBACExists -> k.GetReplicatedAccessPath error: %v", getAccessErr) + continue + } + isExists := false + existsObjectId := "" + walkErr := k.Walk(replicatedAccessPath, "uuid", true, func(node keeper.DumpNode) (bool, error) { + if node.Value == "" { + return false, nil + } + if checkRBACExists(node.Value) { + isExists = true + existsObjectId = strings.TrimPrefix(node.Path, path.Join(replicatedAccessPath, "uuid")+"/") + return true, nil + } + return false, nil + }) + if walkErr != nil { + log.Warn().Msgf("b.isRBACExists -> k.Walk error: %v", walkErr) + continue + } + if isExists { + return true, userDirectory.Name, existsObjectId + } + } + } + return false, "", "" +} + +// https://github.com/Altinity/clickhouse-backup/issues/930 +var needQuoteRBACRE = regexp.MustCompile(`[^0-9a-zA-Z_]`) + +func (b *Backuper) dropExistsRBAC(ctx context.Context, kind string, name string, accessPath string, rbacType, rbacObjectId string, k *keeper.Keeper) error { + //sql + if rbacType == "sql" { + // https://github.com/Altinity/clickhouse-backup/issues/930 + if needQuoteRBACRE.MatchString(name) && !strings.HasPrefix(name, "`") && !strings.HasPrefix(name, `"`) && !strings.HasPrefix(name, "'") && !strings.Contains(name, " ON ") { + name = "`" + name + "`" + } + dropSQL := fmt.Sprintf("DROP %s IF EXISTS %s", kind, name) + return b.ch.QueryContext(ctx, dropSQL) + } + //local + if rbacType == "local" { + return os.Remove(path.Join(accessPath, rbacObjectId+".sql")) + } + //keeper + var keeperPrefixesRBAC = map[string]string{ + "ROLE": "R", + "ROW POLICY": "P", + "SETTINGS PROFILE": "S", + "QUOTA": "Q", + "USER": "U", + } + keeperRBACTypePrefix, isKeeperRBACTypePrefixExists := keeperPrefixesRBAC[kind] + if !isKeeperRBACTypePrefixExists { + return fmt.Errorf("unsupported RBAC kind: %s", kind) + } + prefix, err := k.GetReplicatedAccessPath(rbacType) if err != nil { - return ErrUnknownClickhouseDataPath + return fmt.Errorf("b.dropExistsRBAC -> k.GetReplicatedAccessPath error: %v", err) + } + deletedNodes := []string{ + path.Join(prefix, "uuid", rbacObjectId), + } + walkErr := k.Walk(prefix, keeperRBACTypePrefix, true, func(node keeper.DumpNode) (bool, error) { + if node.Value == rbacObjectId { + deletedNodes = append(deletedNodes, node.Path) + } + return false, nil + }) + if walkErr != nil { + return fmt.Errorf("b.dropExistsRBAC -> k.Walk(%s/%s) error: %v", prefix, keeperRBACTypePrefix, walkErr) + } + + for _, nodePath := range deletedNodes { + if deleteErr := k.Delete(nodePath); deleteErr != nil { + return fmt.Errorf("b.dropExistsRBAC -> k.Delete(%s) error: %v", nodePath, deleteErr) + } + } + return nil +} + +func (b *Backuper) detectRBACObject(sql string) (string, string, error) { + var kind, name string + var detectErr error + + // Define the map of prefixes and their corresponding kinds. + prefixes := map[string]string{ + "ATTACH ROLE": "ROLE", + "ATTACH ROW POLICY": "ROW POLICY", + "ATTACH SETTINGS PROFILE": "SETTINGS PROFILE", + "ATTACH QUOTA": "QUOTA", + "ATTACH USER": "USER", + } + + // Iterate over the prefixes to find a match. + for prefix, k := range prefixes { + if strings.HasPrefix(sql, prefix) { + kind = k + // Extract the name from the SQL query. + name = strings.TrimSpace(strings.TrimPrefix(sql, prefix)) + break + } + } + + // If no match is found, return an error. + if kind == "" { + detectErr = fmt.Errorf("unable to detect RBAC object kind from SQL query: %s", sql) + return kind, name, detectErr + } + names := strings.SplitN(name, " ", 2) + if len(names) > 1 && strings.HasPrefix(names[1], "ON ") { + names = strings.SplitN(name, " ", 4) + name = strings.Join(names[0:3], " ") + } else { + name = names[0] + } + if kind != "ROW POLICY" { + name = strings.Trim(name, "`") + } + name = strings.TrimSpace(name) + if name == "" { + detectErr = fmt.Errorf("unable to detect RBAC object name from SQL query: %s", sql) + return kind, name, detectErr + } + return kind, name, detectErr +} + +// @todo think about restore RBAC from replicated to local *.sql +func (b *Backuper) restoreRBACReplicated(backupName string, backupPrefixDir string, k *keeper.Keeper, replicatedUserDirectories []clickhouse.UserDirectory) error { + if k == nil || len(replicatedUserDirectories) == 0 { + return nil } - srcBackupDir := path.Join(defaultDataPath, "backup", backupName, backupPrefixDir) + srcBackupDir := path.Join(b.DefaultDataPath, "backup", backupName, backupPrefixDir) info, err := os.Stat(srcBackupDir) if err != nil { + log.Warn().Msgf("stat: %s error: %v", srcBackupDir, err) return err } if !info.IsDir() { return fmt.Errorf("%s is not a dir", srcBackupDir) } - replicatedRBAC := make([]struct { - Name string `ch:"name"` - }, 0) - if err = b.ch.SelectContext(ctx, &replicatedRBAC, "SELECT name FROM system.user_directories WHERE type='replicated'"); err == nil && len(replicatedRBAC) > 0 { - jsonLFiles, err := filepathx.Glob(path.Join(srcBackupDir, "*.jsonl")) - if err != nil { - return err + jsonLFiles, err := filepathx.Glob(path.Join(srcBackupDir, "*.jsonl")) + if err != nil { + return err + } + if len(jsonLFiles) == 0 { + return nil + } + restoreReplicatedRBACMap := make(map[string]string, len(jsonLFiles)) + for _, jsonLFile := range jsonLFiles { + for _, userDirectory := range replicatedUserDirectories { + if strings.HasSuffix(jsonLFile, userDirectory.Name+".jsonl") { + restoreReplicatedRBACMap[jsonLFile] = userDirectory.Name + } } - if len(jsonLFiles) == 0 { - return nil + if _, exists := restoreReplicatedRBACMap[jsonLFile]; !exists { + restoreReplicatedRBACMap[jsonLFile] = replicatedUserDirectories[0].Name } - k := keeper.Keeper{Log: log.With().Str("logger", "keeper").Logger()} - if err = k.Connect(ctx, b.ch); err != nil { + } + for jsonLFile, userDirectoryName := range restoreReplicatedRBACMap { + replicatedAccessPath, err := k.GetReplicatedAccessPath(userDirectoryName) + if err != nil { return err } - defer k.Close() - restoreReplicatedRBACMap := make(map[string]string, len(jsonLFiles)) - for _, jsonLFile := range jsonLFiles { - for _, userDirectory := range replicatedRBAC { - if strings.HasSuffix(jsonLFile, userDirectory.Name+".jsonl") { - restoreReplicatedRBACMap[jsonLFile] = userDirectory.Name - } - } - if _, exists := restoreReplicatedRBACMap[jsonLFile]; !exists { - restoreReplicatedRBACMap[jsonLFile] = replicatedRBAC[0].Name - } - } - for jsonLFile, userDirectoryName := range restoreReplicatedRBACMap { - replicatedAccessPath, err := k.GetReplicatedAccessPath(userDirectoryName) - if err != nil { - return err - } - log.Info().Msgf("keeper.Restore(%s) -> %s", jsonLFile, replicatedAccessPath) - if err := k.Restore(jsonLFile, replicatedAccessPath); err != nil { - return err - } + log.Info().Msgf("keeper.Restore(%s) -> %s", jsonLFile, replicatedAccessPath) + if err := k.Restore(jsonLFile, replicatedAccessPath); err != nil { + return err } } return nil @@ -366,25 +776,26 @@ func (b *Backuper) restoreConfigs(backupName string, disks []clickhouse.Disk) er } func (b *Backuper) restoreBackupRelatedDir(backupName, backupPrefixDir, destinationDir string, disks []clickhouse.Disk, skipPatterns []string) error { - defaultDataPath, err := b.ch.GetDefaultPath(disks) - if err != nil { - return ErrUnknownClickhouseDataPath - } - srcBackupDir := path.Join(defaultDataPath, "backup", backupName, backupPrefixDir) + srcBackupDir := path.Join(b.DefaultDataPath, "backup", backupName, backupPrefixDir) info, err := os.Stat(srcBackupDir) if err != nil { + log.Warn().Msgf("stat: %s error: %v", srcBackupDir, err) return err } - + existsFiles, _ := os.ReadDir(destinationDir) + for _, existsF := range existsFiles { + existsI, _ := existsF.Info() + log.Debug().Msgf("%s %v %v", path.Join(destinationDir, existsF.Name()), existsI.Size(), existsI.ModTime()) + } if !info.IsDir() { return fmt.Errorf("%s is not a dir", srcBackupDir) } log.Debug().Msgf("copy %s -> %s", srcBackupDir, destinationDir) copyOptions := recursiveCopy.Options{ - OnDirExists: func(src, dest string) recursiveCopy.DirExistsAction { + OnDirExists: func(src, dst string) recursiveCopy.DirExistsAction { return recursiveCopy.Merge }, - Skip: func(srcinfo os.FileInfo, src, dest string) (bool, error) { + Skip: func(srcinfo os.FileInfo, src, dst string) (bool, error) { for _, pattern := range skipPatterns { if matched, matchErr := filepath.Match(pattern, filepath.Base(src)); matchErr != nil || matched { return true, matchErr @@ -410,60 +821,46 @@ func (b *Backuper) restoreBackupRelatedDir(backupName, backupPrefixDir, destinat return nil } -// RestoreSchema - restore schemas matched by tablePattern from backupName -func (b *Backuper) RestoreSchema(ctx context.Context, backupName, tablePattern string, dropTable, ignoreDependencies bool, disks []clickhouse.Disk) error { - defaultDataPath, err := b.ch.GetDefaultPath(disks) - if err != nil { - return ErrUnknownClickhouseDataPath - } - version, err := b.ch.GetVersion(ctx) - if err != nil { - return err - } - metadataPath := path.Join(defaultDataPath, "backup", backupName, "metadata") - if b.isEmbedded { - defaultDataPath, err = b.ch.GetEmbeddedBackupPath(disks) - if err != nil { - return err +// execute ALTER TABLE db.table DROP PARTITION for corner case when we try to restore backup with the same structure, https://github.com/Altinity/clickhouse-backup/issues/756 +func (b *Backuper) dropExistPartitions(ctx context.Context, tablesForRestore ListOfTables, partitionsIdMap map[metadata.TableTitle][]string, partitions []string, version int) error { + for _, table := range tablesForRestore { + partitionsIds, isExists := partitionsIdMap[metadata.TableTitle{Database: table.Database, Table: table.Table}] + if !isExists { + return fmt.Errorf("`%s`.`%s` doesn't contains %#v partitions", table.Database, table.Table, partitions) } - metadataPath = path.Join(defaultDataPath, backupName, "metadata") - } - info, err := os.Stat(metadataPath) - if err != nil { - return err - } - if !info.IsDir() { - return fmt.Errorf("%s is not a dir", metadataPath) - } - if tablePattern == "" { - tablePattern = "*" - } - tablesForRestore, _, err := b.getTableListByPatternLocal(ctx, metadataPath, tablePattern, dropTable, nil) - if err != nil { - return err - } - // if restore-database-mapping specified, create database in mapping rules instead of in backup files. - if len(b.cfg.General.RestoreDatabaseMapping) > 0 { - err = changeTableQueryToAdjustDatabaseMapping(&tablesForRestore, b.cfg.General.RestoreDatabaseMapping) + partitionsSQL := fmt.Sprintf("DROP PARTITION %s", strings.Join(partitionsIds, ", DROP PARTITION ")) + settings := "" + if version >= 19017000 { + settings = "SETTINGS mutations_sync=2" + } + err := b.ch.QueryContext(ctx, fmt.Sprintf("ALTER TABLE `%s`.`%s` %s %s", table.Database, table.Table, partitionsSQL, settings)) if err != nil { return err } } - if len(tablesForRestore) == 0 { - return fmt.Errorf("no have found schemas by %s in %s", tablePattern, backupName) - } + return nil +} + +// RestoreSchema - restore schemas matched by tablePattern from backupName +func (b *Backuper) RestoreSchema(ctx context.Context, backupName string, backupMetadata metadata.BackupMetadata, disks []clickhouse.Disk, tablesForRestore ListOfTables, ignoreDependencies bool, version int) error { + startRestoreSchema := time.Now() if dropErr := b.dropExistsTables(tablesForRestore, ignoreDependencies, version); dropErr != nil { return dropErr } var restoreErr error if b.isEmbedded { - restoreErr = b.restoreSchemaEmbedded(ctx, backupName, tablesForRestore, defaultDataPath) + restoreErr = b.restoreSchemaEmbedded(ctx, backupName, backupMetadata, disks, tablesForRestore, version) } else { restoreErr = b.restoreSchemaRegular(tablesForRestore, version) } if restoreErr != nil { return restoreErr } + log.Info().Fields(map[string]interface{}{ + "backup": backupName, + "operation": "restore_schema", + "duration": utils.HumanizeDuration(time.Since(startRestoreSchema)), + }).Msg("done") return nil } @@ -471,40 +868,107 @@ var UUIDWithMergeTreeRE = regexp.MustCompile(`^(.+)(UUID)(\s+)'([^']+)'(.+)({uui var emptyReplicatedMergeTreeRE = regexp.MustCompile(`(?m)Replicated(MergeTree|ReplacingMergeTree|SummingMergeTree|AggregatingMergeTree|CollapsingMergeTree|VersionedCollapsingMergeTree|GraphiteMergeTree)\s*\(([^']*)\)(.*)`) -func (b *Backuper) restoreSchemaEmbedded(ctx context.Context, backupName string, tablesForRestore ListOfTables, defaultDataPath string) error { - metadataPath := path.Join(defaultDataPath, backupName, "metadata") - if err := filepath.Walk(metadataPath, func(filePath string, info fs.FileInfo, err error) error { +func (b *Backuper) restoreSchemaEmbedded(ctx context.Context, backupName string, backupMetadata metadata.BackupMetadata, disks []clickhouse.Disk, tablesForRestore ListOfTables, version int) error { + var err error + if tablesForRestore == nil || len(tablesForRestore) == 0 { + if !b.cfg.General.AllowEmptyBackups { + return fmt.Errorf("no tables for restore") + } + log.Warn().Msgf("no tables for restore in embeddded backup %s/metadata.json", backupName) + return nil + } + if b.cfg.ClickHouse.EmbeddedBackupDisk != "" { + err = b.fixEmbeddedMetadataLocal(ctx, backupName, backupMetadata, disks, version) + } else { + err = b.fixEmbeddedMetadataRemote(ctx, backupName, version) + } + if err != nil { + return err + } + return b.restoreEmbedded(ctx, backupName, true, false, version, tablesForRestore, nil) +} + +func (b *Backuper) fixEmbeddedMetadataRemote(ctx context.Context, backupName string, chVersion int) error { + objectDiskPath, err := b.getObjectDiskPath() + if err != nil { + return err + } + if walkErr := b.dst.WalkAbsolute(ctx, path.Join(objectDiskPath, backupName, "metadata"), true, func(ctx context.Context, fInfo storage.RemoteFile) error { if err != nil { return err } - if !strings.HasSuffix(filePath, ".sql") { + if !strings.HasSuffix(fInfo.Name(), ".sql") { return nil } - sqlMetadata, err := object_disk.ReadMetadataFromFile(filePath) + var fReader io.ReadCloser + remoteFilePath := path.Join(objectDiskPath, backupName, "metadata", fInfo.Name()) + fReader, err = b.dst.GetFileReaderAbsolute(ctx, path.Join(objectDiskPath, backupName, "metadata", fInfo.Name())) if err != nil { return err } - sqlBytes, err := object_disk.ReadFileContent(ctx, b.ch, b.cfg, b.cfg.ClickHouse.EmbeddedBackupDisk, filePath) + var sqlBytes []byte + sqlBytes, err = io.ReadAll(fReader) if err != nil { return err } - sqlQuery := string(sqlBytes) - if strings.Contains(sqlQuery, "{uuid}") { - if UUIDWithMergeTreeRE.Match(sqlBytes) { - sqlQuery = UUIDWithMergeTreeRE.ReplaceAllString(sqlQuery, "$1$2$3'$4'$5$4$7") - } else { - log.Warn().Msgf("%s contains `{uuid}` macro, but not contains UUID in table definition, will replace to `{database}/{table}` see https://github.com/ClickHouse/ClickHouse/issues/42709 for details", filePath) - filePathParts := strings.Split(filePath, "/") - database, err := url.QueryUnescape(filePathParts[len(filePathParts)-3]) - if err != nil { + sqlQuery, sqlMetadataChanged, fixSqlErr := b.fixEmbeddedMetadataSQLQuery(ctx, sqlBytes, remoteFilePath, chVersion) + if fixSqlErr != nil { + return fixSqlErr + } + if sqlMetadataChanged { + err = b.dst.PutFileAbsolute(ctx, remoteFilePath, io.NopCloser(strings.NewReader(sqlQuery))) + if err != nil { + return err + } + } + return nil + }); walkErr != nil { + return walkErr + } + return nil +} + +func (b *Backuper) fixEmbeddedMetadataLocal(ctx context.Context, backupName string, backupMetadata metadata.BackupMetadata, disks []clickhouse.Disk, chVersion int) error { + metadataPath := path.Join(b.EmbeddedBackupDataPath, backupName, "metadata") + if walkErr := filepath.Walk(metadataPath, func(filePath string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if !strings.HasSuffix(filePath, ".sql") { + return nil + } + if backupMetadata.DiskTypes[b.cfg.ClickHouse.EmbeddedBackupDisk] == "local" { + sqlBytes, err := os.ReadFile(filePath) + if err != nil { + return err + } + sqlQuery, sqlMetadataChanged, fixSqlErr := b.fixEmbeddedMetadataSQLQuery(ctx, sqlBytes, filePath, chVersion) + if fixSqlErr != nil { + return fixSqlErr + } + if sqlMetadataChanged { + if err = os.WriteFile(filePath, []byte(sqlQuery), 0644); err != nil { return err } - table, err := url.QueryUnescape(filePathParts[len(filePathParts)-2]) - if err != nil { + if err = filesystemhelper.Chown(filePath, b.ch, disks, false); err != nil { return err } - sqlQuery = strings.Replace(sqlQuery, "{uuid}", database+"/"+table, 1) } + return nil + } + sqlMetadata, err := object_disk.ReadMetadataFromFile(filePath) + if err != nil { + return err + } + sqlBytes, err := object_disk.ReadFileContent(ctx, b.ch, b.cfg, b.cfg.ClickHouse.EmbeddedBackupDisk, filePath) + if err != nil { + return err + } + sqlQuery, sqlMetadataChanged, fixSqlErr := b.fixEmbeddedMetadataSQLQuery(ctx, sqlBytes, filePath, chVersion) + if fixSqlErr != nil { + return fixSqlErr + } + if sqlMetadataChanged { if err = object_disk.WriteFileContent(ctx, b.ch, b.cfg, b.cfg.ClickHouse.EmbeddedBackupDisk, filePath, []byte(sqlQuery)); err != nil { return err } @@ -514,28 +978,58 @@ func (b *Backuper) restoreSchemaEmbedded(ctx context.Context, backupName string, return err } } - if emptyReplicatedMergeTreeRE.MatchString(sqlQuery) { - replicaXMLSettings := map[string]string{"default_replica_path": "//default_replica_path", "default_replica_name": "//default_replica_name"} - settings, err := b.ch.GetPreprocessedXMLSettings(ctx, replicaXMLSettings, "config.xml") + return nil + }); walkErr != nil { + return walkErr + } + return nil +} + +func (b *Backuper) fixEmbeddedMetadataSQLQuery(ctx context.Context, sqlBytes []byte, filePath string, version int) (string, bool, error) { + sqlQuery := string(sqlBytes) + sqlMetadataChanged := false + if strings.Contains(sqlQuery, "{uuid}") { + if UUIDWithMergeTreeRE.Match(sqlBytes) && version < 23009000 { + sqlQuery = UUIDWithMergeTreeRE.ReplaceAllString(sqlQuery, "$1$2$3'$4'$5$4$7") + } else { + log.Warn().Msgf("%s contains `{uuid}` macro, will replace to `{database}/{table}` see https://github.com/ClickHouse/ClickHouse/issues/42709 for details", filePath) + filePathParts := strings.Split(filePath, "/") + database, err := url.QueryUnescape(filePathParts[len(filePathParts)-3]) if err != nil { - return err + return "", false, err } - if len(settings) != 2 { - log.Fatal().Stack().Msgf("can't get %#v from preprocessed_configs/config.xml", replicaXMLSettings) + table, err := url.QueryUnescape(filePathParts[len(filePathParts)-2]) + if err != nil { + return "", false, err } - log.Warn().Msgf("%s contains `ReplicatedMergeTree()` without parameters, will replace to '%s` and `%s` see https://github.com/ClickHouse/ClickHouse/issues/42709 for details", filePath, settings["default_replica_path"], settings["default_replica_name"]) - matches := emptyReplicatedMergeTreeRE.FindStringSubmatch(sqlQuery) - substitution := fmt.Sprintf("$1$2('%s','%s')$4", settings["default_replica_path"], settings["default_replica_name"]) - if matches[2] != "" { - substitution = fmt.Sprintf("$1$2('%s','%s',$3)$4", settings["default_replica_path"], settings["default_replica_name"]) + lastIndex := strings.LastIndex(sqlQuery, "{uuid}") + sqlQuery = sqlQuery[:lastIndex] + strings.Replace(sqlQuery[lastIndex:], "{uuid}", database+"/"+table, 1) + // create materialized view corner case + if strings.Contains(sqlQuery, "{uuid}") { + sqlQuery = UUIDWithMergeTreeRE.ReplaceAllString(sqlQuery, "$1$2$3'$4'$5$4$7") } - sqlQuery = emptyReplicatedMergeTreeRE.ReplaceAllString(sqlQuery, substitution) } - return nil - }); err != nil { - return err + sqlMetadataChanged = true + } + if emptyReplicatedMergeTreeRE.MatchString(sqlQuery) { + replicaXMLSettings := map[string]string{"default_replica_path": "//default_replica_path", "default_replica_name": "//default_replica_name"} + settings, err := b.ch.GetPreprocessedXMLSettings(ctx, replicaXMLSettings, "config.xml") + if err != nil { + return "", false, err + } + if len(settings) != 2 { + log.Fatal().Msgf("can't get %#v from preprocessed_configs/config.xml", replicaXMLSettings) + } + log.Warn().Msgf("%s contains `ReplicatedMergeTree()` without parameters, will replace to '%s` and `%s` see https://github.com/ClickHouse/ClickHouse/issues/42709 for details", filePath, settings["default_replica_path"], settings["default_replica_name"]) + matches := emptyReplicatedMergeTreeRE.FindStringSubmatch(sqlQuery) + substitution := fmt.Sprintf("$1$2('%s','%s')$4", settings["default_replica_path"], settings["default_replica_name"]) + if matches[2] != "" { + substitution = fmt.Sprintf("$1$2('%s','%s',$3)$4", settings["default_replica_path"], settings["default_replica_name"]) + } + sqlQuery = emptyReplicatedMergeTreeRE.ReplaceAllString(sqlQuery, substitution) + sqlMetadataChanged = true } - return b.restoreEmbedded(ctx, backupName, true, tablesForRestore, nil) + return sqlQuery, sqlMetadataChanged, nil } func (b *Backuper) restoreSchemaRegular(tablesForRestore ListOfTables, version int) error { @@ -575,7 +1069,7 @@ func (b *Backuper) restoreSchemaRegular(tablesForRestore ListOfTables, version i restoreErr = b.ch.CreateTable(clickhouse.Table{ Database: schema.Database, Name: schema.Table, - }, schema.Query, false, false, b.cfg.General.RestoreSchemaOnCluster, version) + }, schema.Query, false, false, b.cfg.General.RestoreSchemaOnCluster, version, b.DefaultDataPath) if restoreErr != nil { restoreRetries++ @@ -621,16 +1115,17 @@ func (b *Backuper) dropExistsTables(tablesForDrop ListOfTables, ignoreDependenci dropErr = b.ch.DropTable(clickhouse.Table{ Database: schema.Database, Name: schema.Table, - }, query, b.cfg.General.RestoreSchemaOnCluster, ignoreDependencies, version) + }, query, b.cfg.General.RestoreSchemaOnCluster, ignoreDependencies, version, b.DefaultDataPath) if dropErr == nil { tablesForDrop[i].Query = query + break } } } else { dropErr = b.ch.DropTable(clickhouse.Table{ Database: schema.Database, Name: schema.Table, - }, schema.Query, b.cfg.General.RestoreSchemaOnCluster, ignoreDependencies, version) + }, schema.Query, b.cfg.General.RestoreSchemaOnCluster, ignoreDependencies, version, b.DefaultDataPath) } if dropErr != nil { @@ -657,58 +1152,43 @@ func (b *Backuper) dropExistsTables(tablesForDrop ListOfTables, ignoreDependenci } // RestoreData - restore data for tables matched by tablePattern from backupName -func (b *Backuper) RestoreData(ctx context.Context, backupName string, tablePattern string, partitions []string, disks []clickhouse.Disk) error { - startRestore := time.Now() - defaultDataPath, err := b.ch.GetDefaultPath(disks) - if err != nil { - return ErrUnknownClickhouseDataPath - } - if b.ch.IsClickhouseShadow(path.Join(defaultDataPath, "backup", backupName, "shadow")) { - return fmt.Errorf("backups created in v0.0.1 is not supported now") - } - backup, _, err := b.getLocalBackup(ctx, backupName, disks) - if err != nil { - return fmt.Errorf("can't restore: %v", err) - } - +func (b *Backuper) RestoreData(ctx context.Context, backupName string, backupMetadata metadata.BackupMetadata, dataOnly bool, metadataPath, tablePattern string, partitions []string, disks []clickhouse.Disk, version int) error { + var err error + startRestoreData := time.Now() diskMap := make(map[string]string, len(disks)) diskTypes := make(map[string]string, len(disks)) for _, disk := range disks { diskMap[disk.Name] = disk.Path diskTypes[disk.Name] = disk.Type } - for diskName := range backup.Disks { - if _, exists := diskMap[diskName]; !exists { - diskMap[diskName] = backup.Disks[diskName] - } - } - for diskName := range backup.DiskTypes { + for diskName := range backupMetadata.DiskTypes { if _, exists := diskTypes[diskName]; !exists { - diskTypes[diskName] = backup.DiskTypes[diskName] + diskTypes[diskName] = backupMetadata.DiskTypes[diskName] } } var tablesForRestore ListOfTables var partitionsNameList map[metadata.TableTitle][]string - metadataPath := path.Join(defaultDataPath, "backup", backupName, "metadata") - if b.isEmbedded { - metadataPath = path.Join(diskMap[b.cfg.ClickHouse.EmbeddedBackupDisk], backupName, "metadata") - } - if backup.Legacy { - tablesForRestore, err = b.ch.GetBackupTablesLegacy(backupName, disks) - } else { - tablesForRestore, partitionsNameList, err = b.getTableListByPatternLocal(ctx, metadataPath, tablePattern, false, partitions) - } + tablesForRestore, partitionsNameList, err = b.getTableListByPatternLocal(ctx, metadataPath, tablePattern, false, partitions) if err != nil { + // fix https://github.com/Altinity/clickhouse-backup/issues/832 + if b.cfg.General.AllowEmptyBackups && os.IsNotExist(err) { + log.Warn().Msgf("b.getTableListByPatternLocal return error: %v", err) + return nil + } return err } if len(tablesForRestore) == 0 { - return fmt.Errorf("no have found schemas by %s in %s", tablePattern, backupName) + if b.cfg.General.AllowEmptyBackups { + log.Warn().Msgf("not found schemas by %s in %s", tablePattern, backupName) + return nil + } + return fmt.Errorf("not found schemas schemas by %s in %s", tablePattern, backupName) } log.Debug().Msgf("found %d tables with data in backup", len(tablesForRestore)) if b.isEmbedded { - err = b.restoreDataEmbedded(ctx, backupName, tablesForRestore, partitionsNameList) + err = b.restoreDataEmbedded(ctx, backupName, dataOnly, version, tablesForRestore, partitionsNameList) } else { - err = b.restoreDataRegular(ctx, backupName, tablePattern, tablesForRestore, diskMap, diskTypes, disks) + err = b.restoreDataRegular(ctx, backupName, backupMetadata, tablePattern, tablesForRestore, diskMap, diskTypes, disks) } if err != nil { return err @@ -716,199 +1196,319 @@ func (b *Backuper) RestoreData(ctx context.Context, backupName string, tablePatt log.Info().Fields(map[string]interface{}{ "backup": backupName, "operation": "restore_data", - }).Str("duration", utils.HumanizeDuration(time.Since(startRestore))).Msg("done") + }).Str("duration", utils.HumanizeDuration(time.Since(startRestoreData))).Msg("done") return nil } -func (b *Backuper) restoreDataEmbedded(ctx context.Context, backupName string, tablesForRestore ListOfTables, partitionsNameList map[metadata.TableTitle][]string) error { - return b.restoreEmbedded(ctx, backupName, false, tablesForRestore, partitionsNameList) +func (b *Backuper) restoreDataEmbedded(ctx context.Context, backupName string, dataOnly bool, version int, tablesForRestore ListOfTables, partitionsNameList map[metadata.TableTitle][]string) error { + return b.restoreEmbedded(ctx, backupName, false, dataOnly, version, tablesForRestore, partitionsNameList) } -func (b *Backuper) restoreDataRegular(ctx context.Context, backupName string, tablePattern string, tablesForRestore ListOfTables, diskMap, diskTypes map[string]string, disks []clickhouse.Disk) error { +func (b *Backuper) restoreDataRegular(ctx context.Context, backupName string, backupMetadata metadata.BackupMetadata, tablePattern string, tablesForRestore ListOfTables, diskMap, diskTypes map[string]string, disks []clickhouse.Disk) error { if len(b.cfg.General.RestoreDatabaseMapping) > 0 { - tablePattern = b.changeTablePatternFromRestoreDatabaseMapping(tablePattern) + tablePattern = b.changeTablePatternFromRestoreMapping(tablePattern, "database") } + // https://github.com/Altinity/clickhouse-backup/issues/937 + if len(b.cfg.General.RestoreTableMapping) > 0 { + tablePattern = b.changeTablePatternFromRestoreMapping(tablePattern, "table") + } + + if err := b.applyMacrosToObjectDiskPath(ctx); err != nil { + return err + } + chTables, err := b.ch.GetTables(ctx, tablePattern) if err != nil { return err } - disks = b.adjustDisksFromTablesWithSystemDisks(tablesForRestore, diskMap, diskTypes, disks) dstTablesMap := b.prepareDstTablesMap(chTables) missingTables := b.checkMissingTables(tablesForRestore, chTables) if len(missingTables) > 0 { return fmt.Errorf("%s is not created. Restore schema first or create missing tables manually", strings.Join(missingTables, ", ")) } + restoreBackupWorkingGroup, restoreCtx := errgroup.WithContext(ctx) + restoreBackupWorkingGroup.SetLimit(max(b.cfg.ClickHouse.MaxConnections, 1)) - for i, table := range tablesForRestore { - // need mapped database path and original table.Database for HardlinkBackupPartsToStorage + for i := range tablesForRestore { + tableRestoreStartTime := time.Now() + table := tablesForRestore[i] + // need mapped database path and original table.Database for HardlinkBackupPartsToStorage. dstDatabase := table.Database + // The same goes for the table + dstTableName := table.Table if len(b.cfg.General.RestoreDatabaseMapping) > 0 { if targetDB, isMapped := b.cfg.General.RestoreDatabaseMapping[table.Database]; isMapped { dstDatabase = targetDB tablesForRestore[i].Database = targetDB } } + // https://github.com/Altinity/clickhouse-backup/issues/937 + if len(b.cfg.General.RestoreTableMapping) > 0 { + if targetTable, isMapped := b.cfg.General.RestoreTableMapping[table.Table]; isMapped { + dstTableName = targetTable + tablesForRestore[i].Table = targetTable + } + } + logger := log.With().Str("table", fmt.Sprintf("%s.%s", dstDatabase, dstTableName)).Logger() dstTable, ok := dstTablesMap[metadata.TableTitle{ Database: dstDatabase, - Table: table.Table}] + Table: dstTableName}] if !ok { return fmt.Errorf("can't find '%s.%s' in current system.tables", dstDatabase, table.Table) } - // https://github.com/Altinity/clickhouse-backup/issues/529 - if b.cfg.ClickHouse.RestoreAsAttach { - if err = b.restoreDataRegularByAttach(ctx, backupName, table, diskMap, diskTypes, disks, dstTable, tablesForRestore, i); err != nil { - return err - } - } else { - if err = b.restoreDataRegularByParts(ctx, backupName, table, diskMap, diskTypes, disks, dstTable, tablesForRestore, i); err != nil { - return err + idx := i + restoreBackupWorkingGroup.Go(func() error { + // https://github.com/Altinity/clickhouse-backup/issues/529 + if b.cfg.ClickHouse.RestoreAsAttach { + if restoreErr := b.restoreDataRegularByAttach(restoreCtx, backupName, backupMetadata, table, diskMap, diskTypes, disks, dstTable, logger); restoreErr != nil { + return restoreErr + } + } else { + if restoreErr := b.restoreDataRegularByParts(restoreCtx, backupName, backupMetadata, table, diskMap, diskTypes, disks, dstTable, logger); restoreErr != nil { + return restoreErr + } } - } - // https://github.com/Altinity/clickhouse-backup/issues/529 - for _, mutation := range table.Mutations { - if err := b.ch.ApplyMutation(ctx, tablesForRestore[i], mutation); err != nil { - log.Warn().Msgf("can't apply mutation %s for table `%s`.`%s` : %v", mutation.Command, tablesForRestore[i].Database, tablesForRestore[i].Table, err) + // https://github.com/Altinity/clickhouse-backup/issues/529 + for _, mutation := range table.Mutations { + if err := b.ch.ApplyMutation(restoreCtx, tablesForRestore[idx], mutation); err != nil { + log.Warn().Msgf("can't apply mutation %s for table `%s`.`%s` : %v", mutation.Command, tablesForRestore[idx].Database, tablesForRestore[idx].Table, err) + } } - } - log.Info(). - Str("operation", "restore"). - Str("database", table.Database). - Str("table", table.Table). - Msg("done") + log.Info().Fields(map[string]interface{}{ + "duration": utils.HumanizeDuration(time.Since(tableRestoreStartTime)), + "progress": fmt.Sprintf("%d/%d", idx+1, len(tablesForRestore)), + }).Msg("done") + return nil + }) + } + if wgWaitErr := restoreBackupWorkingGroup.Wait(); wgWaitErr != nil { + return fmt.Errorf("one of restoreDataRegular go-routine return error: %v", wgWaitErr) } return nil } -func (b *Backuper) restoreDataRegularByAttach(ctx context.Context, backupName string, table metadata.TableMetadata, diskMap, diskTypes map[string]string, disks []clickhouse.Disk, dstTable clickhouse.Table, tablesForRestore ListOfTables, i int) error { - if err := filesystemhelper.HardlinkBackupPartsToStorage(backupName, table, disks, dstTable.DataPaths, b.ch, false); err != nil { +func (b *Backuper) restoreDataRegularByAttach(ctx context.Context, backupName string, backupMetadata metadata.BackupMetadata, table metadata.TableMetadata, diskMap, diskTypes map[string]string, disks []clickhouse.Disk, dstTable clickhouse.Table, logger zerolog.Logger) error { + if err := filesystemhelper.HardlinkBackupPartsToStorage(backupName, table, disks, diskMap, dstTable.DataPaths, b.ch, false); err != nil { return fmt.Errorf("can't copy data to storage '%s.%s': %v", table.Database, table.Table, err) } - log.Debug().Msg("data to 'storage' copied") - if err := b.downloadObjectDiskParts(ctx, backupName, table, diskMap, diskTypes); err != nil { + logger.Debug().Msg("data to 'storage' copied") + var size int64 + var err error + start := time.Now() + if size, err = b.downloadObjectDiskParts(ctx, backupName, backupMetadata, table, diskMap, diskTypes, disks); err != nil { return fmt.Errorf("can't restore object_disk server-side copy data parts '%s.%s': %v", table.Database, table.Table, err) } - - if err := b.ch.AttachTable(ctx, tablesForRestore[i]); err != nil { - return fmt.Errorf("can't attach table '%s.%s': %v", tablesForRestore[i].Database, tablesForRestore[i].Table, err) + if size > 0 { + logger.Info().Str("duration", utils.HumanizeDuration(time.Since(start))).Str("size", utils.FormatBytes(uint64(size))).Msg("download object_disks finish") + } + if err := b.ch.AttachTable(ctx, table, dstTable); err != nil { + return fmt.Errorf("can't attach table '%s.%s': %v", table.Database, table.Table, err) } return nil } -func (b *Backuper) restoreDataRegularByParts(ctx context.Context, backupName string, table metadata.TableMetadata, diskMap, diskTypes map[string]string, disks []clickhouse.Disk, dstTable clickhouse.Table, tablesForRestore ListOfTables, i int) error { - if err := filesystemhelper.HardlinkBackupPartsToStorage(backupName, table, disks, dstTable.DataPaths, b.ch, true); err != nil { - return fmt.Errorf("can't copy data to datached '%s.%s': %v", table.Database, table.Table, err) +func (b *Backuper) restoreDataRegularByParts(ctx context.Context, backupName string, backupMetadata metadata.BackupMetadata, table metadata.TableMetadata, diskMap, diskTypes map[string]string, disks []clickhouse.Disk, dstTable clickhouse.Table, logger zerolog.Logger) error { + if err := filesystemhelper.HardlinkBackupPartsToStorage(backupName, table, disks, diskMap, dstTable.DataPaths, b.ch, true); err != nil { + return fmt.Errorf("can't copy data to detached '%s.%s': %v", table.Database, table.Table, err) } - log.Debug().Msg("data to 'detached' copied") - if err := b.downloadObjectDiskParts(ctx, backupName, table, diskMap, diskTypes); err != nil { + logger.Debug().Msg("data to 'detached' copied") + logger.Info().Msg("download object_disks start") + var size int64 + var err error + start := time.Now() + if size, err = b.downloadObjectDiskParts(ctx, backupName, backupMetadata, table, diskMap, diskTypes, disks); err != nil { return fmt.Errorf("can't restore object_disk server-side copy data parts '%s.%s': %v", table.Database, table.Table, err) } - if err := b.ch.AttachDataParts(tablesForRestore[i], disks); err != nil { - return fmt.Errorf("can't attach data parts for table '%s.%s': %v", tablesForRestore[i].Database, tablesForRestore[i].Table, err) + log.Info().Str("duration", utils.HumanizeDuration(time.Since(start))).Str("size", utils.FormatBytes(uint64(size))).Msg("download object_disks finish") + if err := b.ch.AttachDataParts(table, dstTable); err != nil { + return fmt.Errorf("can't attach data parts for table '%s.%s': %v", table.Database, table.Table, err) } return nil } -func (b *Backuper) downloadObjectDiskParts(ctx context.Context, backupName string, backupTable metadata.TableMetadata, diskMap, diskTypes map[string]string) error { - start := time.Now() +func (b *Backuper) downloadObjectDiskParts(ctx context.Context, backupName string, backupMetadata metadata.BackupMetadata, backupTable metadata.TableMetadata, diskMap, diskTypes map[string]string, disks []clickhouse.Disk) (int64, error) { + log := apexLog.WithFields(apexLog.Fields{ + "operation": "downloadObjectDiskParts", + "table": fmt.Sprintf("%s.%s", backupTable.Database, backupTable.Table), + }) + size := int64(0) dbAndTableDir := path.Join(common.TablePathEncode(backupTable.Database), common.TablePathEncode(backupTable.Table)) - var err error - needToDownloadObjectDisk := false - for diskName := range backupTable.Parts { - diskType, exists := diskTypes[diskName] - if !exists { - return fmt.Errorf("%s disk doesn't present in diskTypes: %v", diskName, diskTypes) - } - if diskType == "s3" || diskType == "azure_blob_storage" { - needToDownloadObjectDisk = true - break - } - } - if !needToDownloadObjectDisk { - return nil - } - b.dst, err = storage.NewBackupDestination(ctx, b.cfg, b.ch, false, backupName) - if err != nil { - return err - } - if err = b.dst.Connect(ctx); err != nil { - return fmt.Errorf("can't connect to %s: %v", b.dst.Kind(), err) - } - defer func() { - if err := b.dst.Close(ctx); err != nil { - log.Warn().Msgf("downloadObjectDiskParts: can't close BackupDestination error: %v", err) - } - }() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + var err error for diskName, parts := range backupTable.Parts { diskType, exists := diskTypes[diskName] if !exists { - return fmt.Errorf("%s disk doesn't present in diskTypes: %v", diskName, diskTypes) + return 0, fmt.Errorf("%s disk doesn't present in diskTypes: %v", diskName, diskTypes) + } + isObjectDiskEncrypted := false + if diskType == "encrypted" { + if diskPath, exists := diskMap[diskName]; !exists { + for _, part := range parts { + if part.RebalancedDisk != "" { + diskPath = diskMap[part.RebalancedDisk] + if b.isDiskTypeEncryptedObject(clickhouse.Disk{Type: diskTypes[part.RebalancedDisk], Name: part.RebalancedDisk, Path: diskPath}, disks) { + isObjectDiskEncrypted = true + break + } + } + } + } else { + isObjectDiskEncrypted = b.isDiskTypeEncryptedObject(clickhouse.Disk{Type: diskType, Name: diskName, Path: diskPath}, disks) + } } - if diskType == "s3" || diskType == "azure_blob_storage" { + isObjectDisk := b.isDiskTypeObject(diskType) + if isObjectDisk || isObjectDiskEncrypted { if err = config.ValidateObjectDiskConfig(b.cfg); err != nil { - return err + return 0, err } - if err = object_disk.InitCredentialsAndConnections(ctx, b.ch, b.cfg, diskName); err != nil { - return err + if _, exists := diskMap[diskName]; !exists { + for _, part := range parts { + if part.RebalancedDisk != "" { + if err = object_disk.InitCredentialsAndConnections(ctx, b.ch, b.cfg, part.RebalancedDisk); err != nil { + return 0, err + } + } + } + } else if err = object_disk.InitCredentialsAndConnections(ctx, b.ch, b.cfg, diskName); err != nil { + return 0, err } + start := time.Now() + downloadObjectDiskPartsWorkingGroup, downloadCtx := errgroup.WithContext(ctx) + downloadObjectDiskPartsWorkingGroup.SetLimit(int(b.cfg.General.ObjectDiskServerSideCopyConcurrency)) for _, part := range parts { - partPath := path.Join(diskMap[diskName], "backup", backupName, "shadow", dbAndTableDir, diskName, part.Name) - if err := filepath.Walk(partPath, func(fPath string, fInfo fs.FileInfo, err error) error { + dstDiskName := diskName + if part.RebalancedDisk != "" { + dstDiskName = part.RebalancedDisk + } + partPath := path.Join(diskMap[dstDiskName], "backup", backupName, "shadow", dbAndTableDir, dstDiskName, part.Name) + srcBackupName := backupName + srcDiskName := diskName + // copy from required backup for required data parts, https://github.com/Altinity/clickhouse-backup/issues/865 + if part.Required && backupMetadata.RequiredBackup != "" { + var findRecursiveErr error + srcBackupName, srcDiskName, findRecursiveErr = b.findObjectDiskPartRecursive(ctx, backupMetadata, backupTable, part, diskName, log) + if findRecursiveErr != nil { + return 0, findRecursiveErr + } + } + walkErr := filepath.Walk(partPath, func(fPath string, fInfo fs.FileInfo, err error) error { if err != nil { return err } if fInfo.IsDir() { return nil } + // fix https://github.com/Altinity/clickhouse-backup/issues/826 + if strings.Contains(fInfo.Name(), "frozen_metadata") { + return nil + } objMeta, err := object_disk.ReadMetadataFromFile(fPath) if err != nil { return err } - if objMeta.StorageObjectCount < 1 { - return fmt.Errorf("%s: invalid object_dist.Metadata: %#v", fPath, objMeta) + if objMeta.StorageObjectCount < 1 && objMeta.Version < object_disk.VersionRelativePath { + return fmt.Errorf("%s: invalid object_disk.Metadata: %#v", fPath, objMeta) } - var srcBucket, srcKey string - for _, storageObject := range objMeta.StorageObjects { - if b.cfg.General.RemoteStorage == "s3" && diskType == "s3" { - srcBucket = b.cfg.S3.Bucket - srcKey = path.Join(b.cfg.S3.ObjectDiskPath, backupName, diskName, storageObject.ObjectRelativePath) - } else if b.cfg.General.RemoteStorage == "gcs" && diskType == "s3" { - srcBucket = b.cfg.GCS.Bucket - srcKey = path.Join(b.cfg.GCS.ObjectDiskPath, backupName, diskName, storageObject.ObjectRelativePath) - } else if b.cfg.General.RemoteStorage == "azblob" && diskType == "azure_blob_storage" { - srcBucket = b.cfg.AzureBlob.Container - srcKey = path.Join(b.cfg.AzureBlob.ObjectDiskPath, backupName, diskName, storageObject.ObjectRelativePath) - } else { - return fmt.Errorf("incompatible object_disk[%s].Type=%s amd remote_storage: %s", diskName, diskType, b.cfg.General.RemoteStorage) - } - if err = object_disk.CopyObject(ctx, b.ch, b.cfg, diskName, srcBucket, srcKey, storageObject.ObjectRelativePath); err != nil { - return fmt.Errorf("object_disk.CopyObject error: %v", err) + //to allow deleting Object Disk Data during DROP TABLE/DATABASE ...SYNC + if objMeta.RefCount > 0 || objMeta.ReadOnly { + objMeta.RefCount = 0 + objMeta.ReadOnly = false + log.Debugf("%s %#v set RefCount=0 and ReadOnly=0", fPath, objMeta.StorageObjects) + if writeMetaErr := object_disk.WriteMetadataToFile(objMeta, fPath); writeMetaErr != nil { + return fmt.Errorf("%s: object_disk.WriteMetadataToFile return error: %v", fPath, writeMetaErr) } } + downloadObjectDiskPartsWorkingGroup.Go(func() error { + var srcBucket, srcKey string + for _, storageObject := range objMeta.StorageObjects { + if storageObject.ObjectSize == 0 { + continue + } + if b.cfg.General.RemoteStorage == "s3" && (diskType == "s3" || diskType == "encrypted") { + srcBucket = b.cfg.S3.Bucket + srcKey = path.Join(b.cfg.S3.ObjectDiskPath, srcBackupName, srcDiskName, storageObject.ObjectRelativePath) + } else if b.cfg.General.RemoteStorage == "gcs" && (diskType == "s3" || diskType == "encrypted") { + srcBucket = b.cfg.GCS.Bucket + srcKey = path.Join(b.cfg.GCS.ObjectDiskPath, srcBackupName, srcDiskName, storageObject.ObjectRelativePath) + } else if b.cfg.General.RemoteStorage == "azblob" && (diskType == "azure_blob_storage" || diskType == "azure" || diskType == "encrypted") { + srcBucket = b.cfg.AzureBlob.Container + srcKey = path.Join(b.cfg.AzureBlob.ObjectDiskPath, srcBackupName, srcDiskName, storageObject.ObjectRelativePath) + } else { + return fmt.Errorf("incompatible object_disk[%s].Type=%s amd remote_storage: %s", diskName, diskType, b.cfg.General.RemoteStorage) + } + if copiedSize, copyObjectErr := object_disk.CopyObject(downloadCtx, dstDiskName, storageObject.ObjectSize, srcBucket, srcKey, storageObject.ObjectRelativePath); copyObjectErr != nil { + return fmt.Errorf("object_disk.CopyObject error: %v", copyObjectErr) + } else { + atomic.AddInt64(&size, copiedSize) + } + } + return nil + }) return nil - }); err != nil { - return err + }) + if walkErr != nil { + return 0, walkErr } } + if wgWaitErr := downloadObjectDiskPartsWorkingGroup.Wait(); wgWaitErr != nil { + return 0, fmt.Errorf("one of downloadObjectDiskParts go-routine return error: %v", wgWaitErr) + } + log.WithField("disk", diskName).WithField("duration", utils.HumanizeDuration(time.Since(start))).WithField("size", utils.FormatBytes(uint64(size))).Info("object_disk data downloaded") } } - log.Debug().Str("operation", "downloadObjectDiskParts").Str("duration", utils.HumanizeDuration(time.Since(start))).Msg("done") - return nil + + return size, nil +} + +func (b *Backuper) findObjectDiskPartRecursive(ctx context.Context, backup metadata.BackupMetadata, table metadata.TableMetadata, part metadata.Part, diskName string, log *apexLog.Entry) (string, string, error) { + if !part.Required { + return backup.BackupName, diskName, nil + } + if part.Required && backup.RequiredBackup == "" { + return "", "", fmt.Errorf("part %s have required flag, in %s but backup.RequiredBackup is empty", part.Name, backup.BackupName) + } + requiredBackup, err := b.ReadBackupMetadataRemote(ctx, backup.RequiredBackup) + if err != nil { + return "", "", err + } + var requiredTable *metadata.TableMetadata + requiredTable, err = b.downloadTableMetadataIfNotExists(ctx, requiredBackup.BackupName, metadata.TableTitle{Database: table.Database, Table: table.Table}) + // @todo think about add check what if disk type could changed (should already restricted, cause upload seek part in the same disk name) + for requiredDiskName, parts := range requiredTable.Parts { + for _, requiredPart := range parts { + if requiredPart.Name == part.Name { + if requiredPart.Required { + return b.findObjectDiskPartRecursive(ctx, *requiredBackup, *requiredTable, requiredPart, requiredDiskName, log) + } + return requiredBackup.BackupName, requiredDiskName, nil + } + } + + } + return "", "", fmt.Errorf("part %s have required flag in %s, but not found in %s", part.Name, backup.BackupName, backup.RequiredBackup) } func (b *Backuper) checkMissingTables(tablesForRestore ListOfTables, chTables []clickhouse.Table) []string { var missingTables []string for _, table := range tablesForRestore { dstDatabase := table.Database + dstTable := table.Table if len(b.cfg.General.RestoreDatabaseMapping) > 0 { if targetDB, isMapped := b.cfg.General.RestoreDatabaseMapping[table.Database]; isMapped { dstDatabase = targetDB } } + if len(b.cfg.General.RestoreTableMapping) > 0 { + if targetTable, isMapped := b.cfg.General.RestoreTableMapping[table.Table]; isMapped { + dstTable = targetTable + } + } found := false for _, chTable := range chTables { - if (dstDatabase == chTable.Database) && (table.Table == chTable.Name) { + if (dstDatabase == chTable.Database) && (dstTable == chTable.Name) { found = true break } @@ -931,58 +1531,37 @@ func (b *Backuper) prepareDstTablesMap(chTables []clickhouse.Table) map[metadata return dstTablesMap } -func (b *Backuper) adjustDisksFromTablesWithSystemDisks(tablesForRestore ListOfTables, diskMap, diskTypes map[string]string, disks []clickhouse.Disk) []clickhouse.Disk { - for _, t := range tablesForRestore { - for disk := range t.Parts { - if _, diskExists := diskMap[disk]; !diskExists { - if diskTypes[disk] != diskTypes["default"] { - log.Fatal().Msgf("table '%s.%s' require disk '%s' that not found in clickhouse table system.disks, and have different diskType %s than `default` disk %s", t.Database, t.Table, disk, diskTypes[disk], diskTypes["default"]) - } - log.Warn().Msgf("table '%s.%s' require disk '%s' that not found in clickhouse table system.disks, you can add nonexistent disks to `disk_mapping` in `clickhouse` config section, data will restored to %s", t.Database, t.Table, disk, diskMap["default"]) - found := false - for _, d := range disks { - if d.Name == disk { - found = true - break - } - } - if !found { - newDisk := clickhouse.Disk{ - Name: disk, - Path: diskMap["default"], - Type: "local", - } - disks = append(disks, newDisk) - } - } - } - } - return disks -} - -func (b *Backuper) changeTablePatternFromRestoreDatabaseMapping(tablePattern string) string { - for sourceDb, targetDb := range b.cfg.General.RestoreDatabaseMapping { +func (b *Backuper) changeTablePatternFromRestoreMapping(tablePattern, objType string) string { + var mapping map[string]string + switch objType { + case "database": + mapping = b.cfg.General.RestoreDatabaseMapping + case "table": + mapping = b.cfg.General.RestoreDatabaseMapping + default: + return "" + } + for sourceObj, targetObj := range mapping { if tablePattern != "" { - sourceDbRE := regexp.MustCompile(fmt.Sprintf("(^%s.*)|(,%s.*)", sourceDb, sourceDb)) - if sourceDbRE.MatchString(tablePattern) { - matches := sourceDbRE.FindAllStringSubmatch(tablePattern, -1) - substitution := targetDb + ".*" + sourceObjRE := regexp.MustCompile(fmt.Sprintf("(^%s.*)|(,%s.*)", sourceObj, sourceObj)) + if sourceObjRE.MatchString(tablePattern) { + matches := sourceObjRE.FindAllStringSubmatch(tablePattern, -1) + substitution := targetObj + ".*" if strings.HasPrefix(matches[0][1], ",") { substitution = "," + substitution } - tablePattern = sourceDbRE.ReplaceAllString(tablePattern, substitution) + tablePattern = sourceObjRE.ReplaceAllString(tablePattern, substitution) } else { - tablePattern += "," + targetDb + ".*" + tablePattern += "," + targetObj + ".*" } } else { - tablePattern += targetDb + ".*" + tablePattern += targetObj + ".*" } } return tablePattern } -func (b *Backuper) restoreEmbedded(ctx context.Context, backupName string, restoreOnlySchema bool, tablesForRestore ListOfTables, partitionsNameList map[metadata.TableTitle][]string) error { - restoreSQL := "Disk(?,?)" +func (b *Backuper) restoreEmbedded(ctx context.Context, backupName string, schemaOnly, dataOnly bool, version int, tablesForRestore ListOfTables, partitionsNameList map[metadata.TableTitle][]string) error { tablesSQL := "" l := len(tablesForRestore) for i, t := range tablesForRestore { @@ -1003,11 +1582,13 @@ func (b *Backuper) restoreEmbedded(ctx context.Context, backupName string, resto if kind == "TABLE" && len(partitionsNameList) > 0 { if tablePartitions, exists := partitionsNameList[metadata.TableTitle{Table: t.Table, Database: t.Database}]; exists && len(tablePartitions) > 0 { - partitionsSQL := fmt.Sprintf("'%s'", strings.Join(tablePartitions, "','")) - if strings.HasPrefix(partitionsSQL, "'(") { - partitionsSQL = strings.Join(tablePartitions, ",") + if tablePartitions[0] != "*" { + partitionsSQL := fmt.Sprintf("'%s'", strings.Join(tablePartitions, "','")) + if strings.HasPrefix(partitionsSQL, "'(") { + partitionsSQL = strings.Join(tablePartitions, ",") + } + tablesSQL += fmt.Sprintf(" PARTITIONS %s", partitionsSQL) } - tablesSQL += fmt.Sprintf(" PARTITIONS %s", partitionsSQL) } } if i < l-1 { @@ -1015,13 +1596,24 @@ func (b *Backuper) restoreEmbedded(ctx context.Context, backupName string, resto } } } - settings := "" - if restoreOnlySchema { - settings = "SETTINGS structure_only=1" + settings := b.getEmbeddedBackupDefaultSettings(version) + if schemaOnly { + settings = append(settings, "structure_only=1") + } + if dataOnly { + settings = append(settings, "allow_non_empty_tables=1") + } + embeddedBackupLocation, err := b.getEmbeddedBackupLocation(ctx, backupName) + if err != nil { + return err + } + settingsStr := "" + if len(settings) > 0 { + settingsStr = "SETTINGS " + strings.Join(settings, ", ") } - restoreSQL = fmt.Sprintf("RESTORE %s FROM %s %s", tablesSQL, restoreSQL, settings) + restoreSQL := fmt.Sprintf("RESTORE %s FROM %s %s", tablesSQL, embeddedBackupLocation, settingsStr) restoreResults := make([]clickhouse.SystemBackups, 0) - if err := b.ch.SelectContext(ctx, &restoreResults, restoreSQL, b.cfg.ClickHouse.EmbeddedBackupDisk, backupName); err != nil { + if err := b.ch.SelectContext(ctx, &restoreResults, restoreSQL); err != nil { return fmt.Errorf("restore error: %v", err) } if len(restoreResults) == 0 || restoreResults[0].Status != "RESTORED" { diff --git a/pkg/backup/restore_remote.go b/pkg/backup/restore_remote.go index b7781094..9000c35c 100644 --- a/pkg/backup/restore_remote.go +++ b/pkg/backup/restore_remote.go @@ -1,11 +1,13 @@ package backup -func (b *Backuper) RestoreFromRemote(backupName, tablePattern string, databaseMapping, partitions []string, schemaOnly, dataOnly, dropTable, ignoreDependencies, restoreRBAC, rbacOnly, restoreConfigs, configsOnly, resume bool, commandId int) error { - if err := b.Download(backupName, tablePattern, partitions, schemaOnly, resume, commandId); err != nil { +import "errors" + +func (b *Backuper) RestoreFromRemote(backupName, tablePattern string, databaseMapping, tableMapping, partitions []string, schemaOnly, dataOnly, dropExists, ignoreDependencies, restoreRBAC, rbacOnly, restoreConfigs, configsOnly, resume bool, version string, commandId int) error { + if err := b.Download(backupName, tablePattern, partitions, schemaOnly, resume, version, commandId); err != nil { // https://github.com/Altinity/clickhouse-backup/issues/625 - if err != ErrBackupIsAlreadyExists { + if !errors.Is(err, ErrBackupIsAlreadyExists) { return err } } - return b.Restore(backupName, tablePattern, databaseMapping, partitions, schemaOnly, dataOnly, dropTable, ignoreDependencies, restoreRBAC, rbacOnly, restoreConfigs, configsOnly, commandId) + return b.Restore(backupName, tablePattern, databaseMapping, tableMapping, partitions, schemaOnly, dataOnly, dropExists, ignoreDependencies, restoreRBAC, rbacOnly, restoreConfigs, configsOnly, version, commandId) } diff --git a/pkg/backup/restore_test.go b/pkg/backup/restore_test.go new file mode 100644 index 00000000..00400b5f --- /dev/null +++ b/pkg/backup/restore_test.go @@ -0,0 +1,85 @@ +package backup + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestDetectRBACObject(t *testing.T) { + b := &Backuper{} // Create an instance of Backuper for testing + + testCases := []struct { + inputSQL string + expectedKind string + expectedName string + expectedErr error + }{ + { + inputSQL: "ATTACH ROLE `admin`", + expectedKind: "ROLE", + expectedName: "admin", + expectedErr: nil, + }, + { + inputSQL: "ATTACH USER `user1` WITH PASSWORD 'password'", + expectedKind: "USER", + expectedName: "user1", + expectedErr: nil, + }, + { + inputSQL: "ATTACH ROLE test_rbac SETTINGS PROFILE ID('4949fb42-97bb-4841-4b5b-c05d4b0cb685');\n", + expectedKind: "ROLE", + expectedName: "test_rbac", + expectedErr: nil, + }, + { + inputSQL: "ATTACH ROW POLICY `test_rbac` ON default.test_rbac AS restrictive FOR SELECT USING 1 = 1 TO ID('e1469fb8-e014-c22b-4e5c-406134320f91');\n", + expectedKind: "ROW POLICY", + expectedName: "`test_rbac` ON default.test_rbac", + expectedErr: nil, + }, + { + inputSQL: "ATTACH SETTINGS PROFILE `test_rbac` SETTINGS max_execution_time = 60.;\n", + expectedKind: "SETTINGS PROFILE", + expectedName: "test_rbac", + expectedErr: nil, + }, + { + inputSQL: "ATTACH QUOTA test_rbac KEYED BY user_name TO ID('e1469fb8-e014-c22b-4e5c-406134320f91');\n", + expectedKind: "QUOTA", + expectedName: "test_rbac", + expectedErr: nil, + }, + { + inputSQL: "ATTACH USER test_rbac IDENTIFIED WITH sha256_hash BY '256A6D6B157C014A70BE5C62ACA0FE4A6183BFBD45895F62287447B55E519BAD' DEFAULT ROLE ID('2d449952-fca4-c9f2-2949-b83880124bbc');\nATTACH GRANT ID('2d449952-fca4-c9f2-2949-b83880124bbc') TO test_rbac;\n", + expectedKind: "USER", + expectedName: "test_rbac", + expectedErr: nil, + }, + { + inputSQL: "INVALID SQL", + expectedKind: "", + expectedName: "", + expectedErr: fmt.Errorf("unable to detect RBAC object kind from SQL query: INVALID SQL"), + }, + { + inputSQL: "ATTACH USER ", + expectedKind: "USER", + expectedName: "", + expectedErr: fmt.Errorf("unable to detect RBAC object name from SQL query: ATTACH USER "), + }, + } + + for _, tc := range testCases { + kind, name, err := b.detectRBACObject(tc.inputSQL) + assert.Equal(t, tc.expectedKind, kind) + assert.Equal(t, tc.expectedName, name) + if tc.expectedName != "" && tc.expectedKind != "" { + assert.NoError(t, err) + } + if err != nil { + assert.Equal(t, tc.expectedErr.Error(), err.Error()) + } + } +} diff --git a/pkg/backup/table_pattern.go b/pkg/backup/table_pattern.go index a4469afb..dfcf6196 100644 --- a/pkg/backup/table_pattern.go +++ b/pkg/backup/table_pattern.go @@ -13,11 +13,12 @@ import ( "sort" "strings" - "github.com/Altinity/clickhouse-backup/pkg/common" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/filesystemhelper" - "github.com/Altinity/clickhouse-backup/pkg/metadata" - "github.com/Altinity/clickhouse-backup/pkg/partition" + "github.com/Altinity/clickhouse-backup/v2/pkg/common" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/filesystemhelper" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "github.com/Altinity/clickhouse-backup/v2/pkg/partition" + "github.com/google/uuid" "github.com/rs/zerolog/log" ) @@ -129,26 +130,53 @@ func (b *Backuper) getTableListByPatternLocal(ctx context.Context, metadataPath t := result[i] delete(resultPartitionNames, metadata.TableTitle{Database: t.Database, Table: t.Table}) result = append(result[:i], result[i+1:]...) + if i > 0 { + i = i - 1 + } } } return result, resultPartitionNames, nil } +func (b *Backuper) shouldSkipByTableName(tableFullName string) bool { + shallSkipped := false + for _, skipPattern := range b.cfg.ClickHouse.SkipTables { + if shallSkipped, _ = filepath.Match(skipPattern, tableFullName); shallSkipped { + break + } + } + return shallSkipped +} func (b *Backuper) shouldSkipByTableEngine(t metadata.TableMetadata) bool { for _, engine := range b.cfg.ClickHouse.SkipTableEngines { - if engine == "MaterializedView" && (strings.HasPrefix(t.Query, "ATTACH MATERIALIZED VIEW") || strings.HasPrefix(t.Query, "CREATE MATERIALIZED VIEW")) { + //b.log.Debugf("engine=%s query=%s", engine, t.Query) + if strings.ToLower(engine) == "dictionary" && (strings.HasPrefix(t.Query, "ATTACH DICTIONARY") || strings.HasPrefix(t.Query, "CREATE DICTIONARY")) { log.Warn().Msgf("shouldSkipByTableEngine engine=%s found in : %s", engine, t.Query) return true } - if engine == "View" && strings.HasPrefix(t.Query, "CREATE VIEW") { + if strings.ToLower(engine) == "materializedview" && (strings.HasPrefix(t.Query, "ATTACH MATERIALIZED VIEW") || strings.HasPrefix(t.Query, "CREATE MATERIALIZED VIEW")) { log.Warn().Msgf("shouldSkipByTableEngine engine=%s found in : %s", engine, t.Query) return true } - if shouldSkip, err := regexp.MatchString(fmt.Sprintf("(?mi)ENGINE\\s*=\\s*%s\\(", engine), t.Query); err == nil && shouldSkip { + if strings.ToLower(engine) == "view" && (strings.HasPrefix(t.Query, "ATTACH VIEW") || strings.HasPrefix(t.Query, "CREATE VIEW")) { log.Warn().Msgf("shouldSkipByTableEngine engine=%s found in : %s", engine, t.Query) return true - } else if err != nil { - log.Warn().Msgf("shouldSkipByTableEngine engine=%s return error: %v", engine, err) + } + if strings.ToLower(engine) == "liveview" && (strings.HasPrefix(t.Query, "ATTACH LIVE") || strings.HasPrefix(t.Query, "CREATE LIVE")) { + log.Warn().Msgf("shouldSkipByTableEngine engine=%s found in : %s", engine, t.Query) + return true + } + if strings.ToLower(engine) == "windowview" && (strings.HasPrefix(t.Query, "ATTACH WINDOW") || strings.HasPrefix(t.Query, "CREATE WINDOW")) { + log.Warn().Msgf("shouldSkipByTableEngine engine=%s found in : %s", engine, t.Query) + return true + } + if engine != "" { + if shouldSkip, err := regexp.MatchString(fmt.Sprintf("(?mi)ENGINE\\s*=\\s*%s([\\(\\s]|\\s*)", engine), t.Query); err == nil && shouldSkip { + log.Warn().Msgf("shouldSkipByTableEngine engine=%s found in : %s", engine, t.Query) + return true + } else if err != nil { + log.Warn().Msgf("shouldSkipByTableEngine engine=%s return error: %v", engine, err) + } } } return false @@ -166,12 +194,7 @@ func (b *Backuper) checkShallSkipped(p string, metadataPath string) ([]string, s } table, _ := url.PathUnescape(names[1]) tableFullName := fmt.Sprintf("%s.%s", database, table) - shallSkipped := false - for _, skipPattern := range b.cfg.ClickHouse.SkipTables { - if shallSkipped, _ = filepath.Match(skipPattern, tableFullName); shallSkipped { - break - } - } + shallSkipped := b.shouldSkipByTableName(tableFullName) return names, database, table, tableFullName, shallSkipped, true } @@ -245,6 +268,7 @@ func (b *Backuper) enrichTablePatternsByInnerDependencies(metadataPath string, t innerTableFile = path.Join(innerTableFile, common.TablePathEncode(fmt.Sprintf(".inner.%s", table))) innerTableName += fmt.Sprintf(".inner.%s", table) } + // https://github.com/Altinity/clickhouse-backup/issues/765, .inner. table could be dropped manually, .inner. table is required for ATTACH if _, err := os.Stat(path.Join(metadataPath, innerTableFile+".json")); err != nil { return err } @@ -270,12 +294,13 @@ func (b *Backuper) enrichTablePatternsByInnerDependencies(metadataPath string, t return tablePatterns, nil } -var queryRE = regexp.MustCompile(`(?m)^(CREATE|ATTACH) (TABLE|VIEW|LIVE VIEW|MATERIALIZED VIEW|DICTIONARY|FUNCTION) (\x60?)([^\s\x60.]*)(\x60?)\.([^\s\x60.]*)(?:( UUID '[^']+'))?(?:( TO )(\x60?)([^\s\x60.]*)(\x60?)(\.))?(?:(.+FROM )(\x60?)([^\s\x60.]*)(\x60?)(\.))?`) +var queryRE = regexp.MustCompile(`(?m)^(CREATE|ATTACH) (TABLE|VIEW|LIVE VIEW|MATERIALIZED VIEW|DICTIONARY|FUNCTION) (\x60?)([^\s\x60.]*)(\x60?)\.\x60?([^\s\x60.]*)\x60?( UUID '[^']+')?(?:( TO )(\x60?)([^\s\x60.]*)(\x60?)(\.)(\x60?)([^\s\x60.]*)(\x60?))?(?:(.+FROM )(\x60?)([^\s\x60.]*)(\x60?)(\.)(\x60?)([^\s\x60.]*)(\x60?))?`) var createOrAttachRE = regexp.MustCompile(`(?m)^(CREATE|ATTACH)`) var uuidRE = regexp.MustCompile(`UUID '([a-f\d\-]+)'`) +var usualIdentifier = regexp.MustCompile(`^[a-zA-Z0-9_]+$`) var replicatedRE = regexp.MustCompile(`(Replicated[a-zA-Z]*MergeTree)\('([^']+)'([^)]+)\)`) -var distributedRE = regexp.MustCompile(`(Distributed)\(([^,]+),([^,]+),([^)]+)\)`) +var distributedRE = regexp.MustCompile(`(Distributed)\(([^,]+),([^,]+),([^,]+)([^)]+)\)`) func changeTableQueryToAdjustDatabaseMapping(originTables *ListOfTables, dbMapRule map[string]string) error { for i := 0; i < len(*originTables); i++ { @@ -292,13 +317,20 @@ func changeTableQueryToAdjustDatabaseMapping(originTables *ListOfTables, dbMapRu setMatchedDb := func(clauseTargetDb string) string { if clauseMappedDb, isClauseMapped := dbMapRule[clauseTargetDb]; isClauseMapped { clauseTargetDb = clauseMappedDb + if !usualIdentifier.MatchString(clauseTargetDb) { + clauseTargetDb = "`" + clauseTargetDb + "`" + } } return clauseTargetDb } + createTargetDb := targetDB + if !usualIdentifier.MatchString(createTargetDb) { + createTargetDb = "`" + createTargetDb + "`" + } toClauseTargetDb := setMatchedDb(matches[0][10]) - fromClauseTargetDb := setMatchedDb(matches[0][15]) + fromClauseTargetDb := setMatchedDb(matches[0][18]) // matching CREATE|ATTACH ... TO .. SELECT ... FROM ... command - substitution = fmt.Sprintf("${1} ${2} ${3}%v${5}.${6}${7}${8}${9}%v${11}${12}${13}${14}%v${16}${17}", targetDB, toClauseTargetDb, fromClauseTargetDb) + substitution = fmt.Sprintf("${1} ${2} ${3}%v${5}.${6}${7}${8}${9}%v${11}${12}${13}${14}${15}${16}${17}%v${19}${20}${21}${22}${23}", createTargetDb, toClauseTargetDb, fromClauseTargetDb) } else { if originTable.Query == "" { continue @@ -306,7 +338,7 @@ func changeTableQueryToAdjustDatabaseMapping(originTables *ListOfTables, dbMapRu return fmt.Errorf("error when try to replace database `%s` to `%s` in query: %s", originTable.Database, targetDB, originTable.Query) } originTable.Query = queryRE.ReplaceAllString(originTable.Query, substitution) - if len(uuidRE.FindAllString(originTable.Query, -1)) > 0 { + if uuidRE.MatchString(originTable.Query) { newUUID, _ := uuid.NewUUID() substitution = fmt.Sprintf("UUID '%s'", newUUID.String()) originTable.Query = uuidRE.ReplaceAllString(originTable.Query, substitution) @@ -327,7 +359,7 @@ func changeTableQueryToAdjustDatabaseMapping(originTables *ListOfTables, dbMapRu underlyingDB := matches[0][3] underlyingDBClean := strings.NewReplacer(" ", "", "'", "").Replace(underlyingDB) if underlyingTargetDB, isUnderlyingMapped := dbMapRule[underlyingDBClean]; isUnderlyingMapped { - substitution = fmt.Sprintf("${1}(${2},%s,${4})", strings.Replace(underlyingDB, underlyingDBClean, underlyingTargetDB, 1)) + substitution = fmt.Sprintf("${1}(${2},%s,${4}${5})", strings.Replace(underlyingDB, underlyingDBClean, underlyingTargetDB, 1)) originTable.Query = distributedRE.ReplaceAllString(originTable.Query, substitution) } } @@ -338,6 +370,74 @@ func changeTableQueryToAdjustDatabaseMapping(originTables *ListOfTables, dbMapRu return nil } +func changeTableQueryToAdjustTableMapping(originTables *ListOfTables, tableMapRule map[string]string) error { + for i := 0; i < len(*originTables); i++ { + originTable := (*originTables)[i] + if targetTable, isMapped := tableMapRule[originTable.Table]; isMapped { + // substitute table in the table create query + var substitution string + + if createOrAttachRE.MatchString(originTable.Query) { + matches := queryRE.FindAllStringSubmatch(originTable.Query, -1) + if matches[0][6] != originTable.Table { + return fmt.Errorf("invalid SQL: %s for restore-table-mapping[%s]=%s", originTable.Query, originTable.Table, targetTable) + } + setMatchedDb := func(clauseTargetTable string) string { + if clauseMappedTable, isClauseMapped := tableMapRule[clauseTargetTable]; isClauseMapped { + clauseTargetTable = clauseMappedTable + if !usualIdentifier.MatchString(clauseTargetTable) { + clauseTargetTable = "`" + clauseTargetTable + "`" + } + } + return clauseTargetTable + } + createTargetTable := targetTable + if !usualIdentifier.MatchString(createTargetTable) { + createTargetTable = "`" + createTargetTable + "`" + } + toClauseTargetTable := setMatchedDb(matches[0][14]) + fromClauseTargetTable := setMatchedDb(matches[0][22]) + // matching CREATE|ATTACH ... TO .. SELECT ... FROM ... command + substitution = fmt.Sprintf("${1} ${2} ${3}${4}${5}.%v${7}${8}${9}${10}${11}${12}${13}%v${15}${16}${17}${18}${19}${20}${21}%v${23}", createTargetTable, toClauseTargetTable, fromClauseTargetTable) + } else { + if originTable.Query == "" { + continue + } + return fmt.Errorf("error when try to replace table `%s` to `%s` in query: %s", originTable.Table, targetTable, originTable.Query) + } + originTable.Query = queryRE.ReplaceAllString(originTable.Query, substitution) + if uuidRE.MatchString(originTable.Query) { + newUUID, _ := uuid.NewUUID() + substitution = fmt.Sprintf("UUID '%s'", newUUID.String()) + originTable.Query = uuidRE.ReplaceAllString(originTable.Query, substitution) + } + // https://github.com/Altinity/clickhouse-backup/issues/547 + if replicatedRE.MatchString(originTable.Query) { + matches := replicatedRE.FindAllStringSubmatch(originTable.Query, -1) + originPath := matches[0][2] + tableReplicatedPattern := "/" + originTable.Table + if strings.Contains(originPath, tableReplicatedPattern) { + substitution = fmt.Sprintf("${1}('%s'${3})", strings.Replace(originPath, tableReplicatedPattern, "/"+targetTable, 1)) + originTable.Query = replicatedRE.ReplaceAllString(originTable.Query, substitution) + } + } + // https://github.com/Altinity/clickhouse-backup/issues/547 + if distributedRE.MatchString(originTable.Query) { + matches := distributedRE.FindAllStringSubmatch(originTable.Query, -1) + underlyingTable := matches[0][4] + underlyingTableClean := strings.NewReplacer(" ", "", "'", "").Replace(underlyingTable) + if underlyingTargetTable, isUnderlyingMapped := tableMapRule[underlyingTableClean]; isUnderlyingMapped { + substitution = fmt.Sprintf("${1}(${2},${3},%s${5})", strings.Replace(underlyingTable, underlyingTableClean, underlyingTargetTable, 1)) + originTable.Query = distributedRE.ReplaceAllString(originTable.Query, substitution) + } + } + originTable.Table = targetTable + (*originTables)[i] = originTable + } + } + return nil +} + func filterPartsAndFilesByPartitionsFilter(tableMetadata metadata.TableMetadata, partitionsFilter common.EmptyMap) { if len(partitionsFilter) > 0 { for disk, parts := range tableMetadata.Parts { @@ -374,11 +474,8 @@ func getTableListByPatternRemote(ctx context.Context, b *Backuper, remoteBackupM continue } tableName := fmt.Sprintf("%s.%s", t.Database, t.Table) - shallSkipped := false - for _, skipPattern := range b.cfg.ClickHouse.SkipTables { - if shallSkipped, _ = filepath.Match(skipPattern, tableName); shallSkipped { - break - } + if shallSkipped := b.shouldSkipByTableName(tableName); shallSkipped { + continue } tablePatterns: for _, p := range tablePatterns { @@ -386,7 +483,7 @@ func getTableListByPatternRemote(ctx context.Context, b *Backuper, remoteBackupM case <-ctx.Done(): return nil, ctx.Err() default: - if matched, _ := filepath.Match(strings.Trim(p, " \t\r\n"), tableName); !matched || shallSkipped { + if matched, _ := filepath.Match(strings.Trim(p, " \t\r\n"), tableName); !matched { continue } tmReader, err := b.dst.GetFileReader(ctx, path.Join(metadataPath, common.TablePathEncode(t.Database), fmt.Sprintf("%s.json", common.TablePathEncode(t.Table)))) @@ -414,8 +511,15 @@ func getTableListByPatternRemote(ctx context.Context, b *Backuper, remoteBackupM return result, nil } +var lowPriorityEnginesRE = regexp.MustCompile(`ENGINE = (Distributed|Dictionary|Merge)\(`) +var streamingEnginesRE = regexp.MustCompile(`ENGINE = (Kafka|NATS|RabbitMQ|S3Queue)`) + func getOrderByEngine(query string, dropTable bool) int64 { - if strings.Contains(query, "ENGINE = Distributed") || strings.Contains(query, "ENGINE = Kafka") || strings.Contains(query, "ENGINE = RabbitMQ") { + if lowPriorityEnginesRE.MatchString(query) { + return 5 + } + + if streamingEnginesRE.MatchString(query) { return 4 } if strings.HasPrefix(query, "CREATE DICTIONARY") { @@ -481,6 +585,10 @@ func ShallSkipDatabase(cfg *config.Config, targetDB, tablePattern string) bool { if strings.HasSuffix(pattern, ".*") && strings.TrimSuffix(pattern, ".*") == targetDB { return false } + // https://github.com/Altinity/clickhouse-backup/issues/663 + if matched, err := filepath.Match(pattern, targetDB+"."); err == nil && matched { + return false + } } return true } @@ -490,7 +598,8 @@ func ShallSkipDatabase(cfg *config.Config, targetDB, tablePattern string) bool { skipTablesPatterns = append(skipTablesPatterns, cfg.ClickHouse.SkipTables...) for _, pattern := range skipTablesPatterns { pattern = strings.Trim(pattern, " \r\t\n") - if strings.HasSuffix(pattern, ".*") && strings.TrimSuffix(pattern, ".*") == targetDB { + // https://github.com/Altinity/clickhouse-backup/issues/663 + if matched, err := filepath.Match(pattern, targetDB+"."); err == nil && matched { return true } } diff --git a/pkg/backup/upload.go b/pkg/backup/upload.go index 1f909e4f..56841695 100644 --- a/pkg/backup/upload.go +++ b/pkg/backup/upload.go @@ -5,11 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/custom" - "github.com/Altinity/clickhouse-backup/pkg/resumable" - "github.com/Altinity/clickhouse-backup/pkg/status" - "github.com/eapache/go-resiliency/retrier" "io" "os" "path" @@ -20,18 +15,24 @@ import ( "sync/atomic" "time" + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/custom" + "github.com/Altinity/clickhouse-backup/v2/pkg/resumable" + "github.com/Altinity/clickhouse-backup/v2/pkg/status" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage" + "github.com/eapache/go-resiliency/retrier" + "golang.org/x/sync/errgroup" - "golang.org/x/sync/semaphore" - "github.com/Altinity/clickhouse-backup/pkg/common" - "github.com/Altinity/clickhouse-backup/pkg/filesystemhelper" - "github.com/Altinity/clickhouse-backup/pkg/metadata" - "github.com/Altinity/clickhouse-backup/pkg/utils" + "github.com/Altinity/clickhouse-backup/v2/pkg/common" + "github.com/Altinity/clickhouse-backup/v2/pkg/filesystemhelper" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" "github.com/rs/zerolog/log" "github.com/yargevad/filepathx" ) -func (b *Backuper) Upload(backupName, diffFrom, diffFromRemote, tablePattern string, partitions []string, schemaOnly, resume bool, commandId int) error { +func (b *Backuper) Upload(backupName string, deleteSource bool, diffFrom, diffFromRemote, tablePattern string, partitions []string, schemaOnly, resume bool, backupVersion string, commandId int) error { ctx, cancel, err := status.Current.GetContextWithCancel(commandId) if err != nil { return err @@ -59,7 +60,7 @@ func (b *Backuper) Upload(backupName, diffFrom, diffFromRemote, tablePattern str if _, disks, err = b.getLocalBackup(ctx, backupName, nil); err != nil { return fmt.Errorf("can't find local backup: %v", err) } - if err := b.init(ctx, disks, backupName); err != nil { + if err := b.initDisksPathsAndBackupDestination(ctx, disks, backupName); err != nil { return err } defer func() { @@ -70,7 +71,7 @@ func (b *Backuper) Upload(backupName, diffFrom, diffFromRemote, tablePattern str remoteBackups, err := b.dst.BackupList(ctx, false, "") if err != nil { - return err + return fmt.Errorf("b.dst.BackupList return error: %v", err) } for i := range remoteBackups { if backupName == remoteBackups[i].BackupName { @@ -83,7 +84,7 @@ func (b *Backuper) Upload(backupName, diffFrom, diffFromRemote, tablePattern str } backupMetadata, err := b.ReadBackupMetadataLocal(ctx, backupName) if err != nil { - return err + return fmt.Errorf("b.ReadBackupMetadataLocal return error: %v", err) } var tablesForUpload ListOfTables b.isEmbedded = strings.Contains(backupMetadata.Tags, "embedded") @@ -94,22 +95,24 @@ func (b *Backuper) Upload(backupName, diffFrom, diffFromRemote, tablePattern str if len(backupMetadata.Tables) != 0 { tablesForUpload, err = b.prepareTableListToUpload(ctx, backupName, tablePattern, partitions) if err != nil { - return err + return fmt.Errorf("b.prepareTableListToUpload return error: %v", err) } } tablesForUploadFromDiff := map[metadata.TableTitle]metadata.TableMetadata{} if diffFrom != "" && !b.isEmbedded { - tablesForUploadFromDiff, err = b.getTablesForUploadDiffLocal(ctx, diffFrom, backupMetadata, tablePattern) + tablesForUploadFromDiff, err = b.getTablesDiffFromLocal(ctx, diffFrom, tablePattern) if err != nil { - return err + return fmt.Errorf("b.getTablesDiffFromLocal return error: %v", err) } + backupMetadata.RequiredBackup = diffFrom } if diffFromRemote != "" && !b.isEmbedded { - tablesForUploadFromDiff, err = b.getTablesForUploadDiffRemote(ctx, diffFromRemote, backupMetadata, tablePattern) + tablesForUploadFromDiff, err = b.getTablesDiffFromRemote(ctx, diffFromRemote, tablePattern) if err != nil { - return err + return fmt.Errorf("b.getTablesDiffFromRemote return error: %v", err) } + backupMetadata.RequiredBackup = diffFromRemote } if b.resume { b.resumableState = resumable.NewState(b.DefaultDataPath, backupName, "upload", map[string]interface{}{ @@ -125,14 +128,10 @@ func (b *Backuper) Upload(backupName, diffFrom, diffFromRemote, tablePattern str metadataSize := int64(0) log.Debug().Msgf("prepare table concurrent semaphore with concurrency=%d len(tablesForUpload)=%d", b.cfg.General.UploadConcurrency, len(tablesForUpload)) - uploadSemaphore := semaphore.NewWeighted(int64(b.cfg.General.UploadConcurrency)) uploadGroup, uploadCtx := errgroup.WithContext(ctx) + uploadGroup.SetLimit(int(b.cfg.General.UploadConcurrency)) for i, table := range tablesForUpload { - if err := uploadSemaphore.Acquire(uploadCtx, 1); err != nil { - log.Error().Msgf("can't acquire semaphore during Upload table: %v", err) - break - } start := time.Now() if !schemaOnly { if diffTable, diffExists := tablesForUploadFromDiff[metadata.TableTitle{ @@ -145,27 +144,30 @@ func (b *Backuper) Upload(backupName, diffFrom, diffFromRemote, tablePattern str } idx := i uploadGroup.Go(func() error { - defer uploadSemaphore.Release(1) var uploadedBytes int64 - if !schemaOnly { + //skip upload data for embedded backup with empty embedded_backup_disk + if !schemaOnly && (!b.isEmbedded || b.cfg.ClickHouse.EmbeddedBackupDisk != "") { var files map[string][]string var err error - files, uploadedBytes, err = b.uploadTableData(uploadCtx, backupName, tablesForUpload[idx]) + files, uploadedBytes, err = b.uploadTableData(uploadCtx, backupName, deleteSource, tablesForUpload[idx]) if err != nil { return err } atomic.AddInt64(&compressedDataSize, uploadedBytes) tablesForUpload[idx].Files = files } - tableMetadataSize, err := b.uploadTableMetadata(uploadCtx, backupName, tablesForUpload[idx]) + tableMetadataSize, err := b.uploadTableMetadata(uploadCtx, backupName, backupMetadata.RequiredBackup, tablesForUpload[idx]) if err != nil { return err } atomic.AddInt64(&metadataSize, tableMetadataSize) - log.Info(). - Str("table", fmt.Sprintf("%s.%s", tablesForUpload[idx].Database, tablesForUpload[idx].Table)).Str("duration", utils.HumanizeDuration(time.Since(start))). - Str("size", utils.FormatBytes(uint64(uploadedBytes+tableMetadataSize))). - Msg("done") + log.Info().Fields(map[string]interface{}{ + "table": fmt.Sprintf("%s.%s", tablesForUpload[idx].Database, tablesForUpload[idx].Table), + "progress": fmt.Sprintf("%d/%d", idx+1, len(tablesForUpload)), + "duration": utils.HumanizeDuration(time.Since(start)), + "size": utils.FormatBytes(uint64(uploadedBytes + tableMetadataSize)), + "version": backupVersion, + }).Msg("done") return nil }) } @@ -173,16 +175,24 @@ func (b *Backuper) Upload(backupName, diffFrom, diffFromRemote, tablePattern str return fmt.Errorf("one of upload table go-routine return error: %v", err) } - if !b.isEmbedded { - // upload rbac for backup - if backupMetadata.RBACSize, err = b.uploadRBACData(ctx, backupName); err != nil { - return err - } + // upload rbac for backup + if backupMetadata.RBACSize, err = b.uploadRBACData(ctx, backupName); err != nil { + return fmt.Errorf("b.uploadRBACData return error: %v", err) + } - // upload configs for backup - if backupMetadata.ConfigSize, err = b.uploadConfigData(ctx, backupName); err != nil { - return err + // upload configs for backup + if backupMetadata.ConfigSize, err = b.uploadConfigData(ctx, backupName); err != nil { + return fmt.Errorf("b.uploadConfigData return error: %v", err) + } + //upload embedded .backup file + if b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk != "" && backupMetadata.Tables != nil && len(backupMetadata.Tables) > 0 { + localClickHouseBackupFile := path.Join(b.EmbeddedBackupDataPath, backupName, ".backup") + remoteClickHouseBackupFile := path.Join(backupName, ".backup") + localEmbeddedMetadataSize := int64(0) + if localEmbeddedMetadataSize, err = b.uploadSingleBackupFile(ctx, localClickHouseBackupFile, remoteClickHouseBackupFile); err != nil { + return fmt.Errorf("b.uploadSingleBackupFile return error: %v", err) } + metadataSize += localEmbeddedMetadataSize } // upload metadata for backup @@ -201,6 +211,7 @@ func (b *Backuper) Upload(backupName, diffFrom, diffFromRemote, tablePattern str } else { backupMetadata.DataFormat = DirectoryFormat } + backupMetadata.ClickhouseBackupVersion = backupVersion newBackupMetadataBody, err := json.MarshalIndent(backupMetadata, "", "\t") if err != nil { return err @@ -215,39 +226,81 @@ func (b *Backuper) Upload(backupName, diffFrom, diffFromRemote, tablePattern str return fmt.Errorf("can't upload %s: %v", remoteBackupMetaFile, err) } } - if b.isEmbedded { - localClickHouseBackupFile := path.Join(b.EmbeddedBackupDataPath, backupName, ".backup") - remoteClickHouseBackupFile := path.Join(backupName, ".backup") - if err = b.uploadSingleBackupFile(ctx, localClickHouseBackupFile, remoteClickHouseBackupFile); err != nil { - return err - } - } if b.resume { b.resumableState.Close() } - log.Info(). - Fields(map[string]interface{}{ - "backup": backupName, - "operation": "upload", - }). - Str("duration", utils.HumanizeDuration(time.Since(startUpload))). - Str("size", utils.FormatBytes(uint64(compressedDataSize)+uint64(metadataSize)+uint64(len(newBackupMetadataBody))+backupMetadata.RBACSize+backupMetadata.ConfigSize)). - Msg("done") - - // Clean - if err = b.dst.RemoveOldBackups(ctx, b.cfg.General.BackupsToKeepRemote); err != nil { + log.Info().Fields(map[string]interface{}{ + "backup": backupName, + "operation": "upload", + "duration": utils.HumanizeDuration(time.Since(startUpload)), + "upload_size": utils.FormatBytes(uint64(compressedDataSize) + uint64(metadataSize) + uint64(len(newBackupMetadataBody)) + backupMetadata.RBACSize + backupMetadata.ConfigSize), + "object_disk_size": utils.FormatBytes(backupMetadata.ObjectDiskSize), + "version": backupVersion, + }).Msg("done") + + // Remote old backup retention + if err = b.RemoveOldBackupsRemote(ctx); err != nil { return fmt.Errorf("can't remove old backups on remote storage: %v", err) } + // Local old backup retention, fix https://github.com/Altinity/clickhouse-backup/issues/834 + if err = b.RemoveOldBackupsLocal(ctx, false, nil); err != nil { + return fmt.Errorf("can't remove old local backups: %v", err) + } + + // explicitly delete local backup after successful upload, fix https://github.com/Altinity/clickhouse-backup/issues/777 + if b.cfg.General.BackupsToKeepLocal >= 0 && deleteSource { + if err = b.RemoveBackupLocal(ctx, backupName, disks); err != nil { + return fmt.Errorf("can't explicitly delete local source backup: %v", err) + } + } return nil } -func (b *Backuper) uploadSingleBackupFile(ctx context.Context, localFile, remoteFile string) error { - if b.resume && b.resumableState.IsAlreadyProcessedBool(remoteFile) { +func (b *Backuper) RemoveOldBackupsRemote(ctx context.Context) error { + + if b.cfg.General.BackupsToKeepRemote < 1 { return nil } + start := time.Now() + backupList, err := b.dst.BackupList(ctx, true, "") + if err != nil { + return err + } + backupsToDelete := storage.GetBackupsToDeleteRemote(backupList, b.cfg.General.BackupsToKeepRemote) + log.Info().Fields(map[string]interface{}{ + "operation": "RemoveOldBackupsRemote", + "duration": utils.HumanizeDuration(time.Since(start)), + }).Msg("calculate backup list for delete remote") + for _, backupToDelete := range backupsToDelete { + startDelete := time.Now() + err = b.cleanEmbeddedAndObjectDiskRemoteIfSameLocalNotPresent(ctx, backupToDelete) + if err != nil { + return err + } + + if err := b.dst.RemoveBackupRemote(ctx, backupToDelete); err != nil { + log.Warn().Msgf("can't deleteKey %s return error : %v", backupToDelete.BackupName, err) + } + log.Info().Fields(map[string]interface{}{ + "operation": "RemoveOldBackupsRemote", + "location": "remote", + "backup": backupToDelete.BackupName, + "duration": utils.HumanizeDuration(time.Since(startDelete)), + }).Msg("done") + } + log.Info().Fields(map[string]interface{}{"operation": "RemoveOldBackupsRemote", "duration": utils.HumanizeDuration(time.Since(start))}).Msg("done") + return nil +} + +func (b *Backuper) uploadSingleBackupFile(ctx context.Context, localFile, remoteFile string) (int64, error) { + if b.resume { + if isProcessed, size := b.resumableState.IsAlreadyProcessed(remoteFile); isProcessed { + return size, nil + } + } f, err := os.Open(localFile) if err != nil { - return fmt.Errorf("can't open %s: %v", localFile, err) + return 0, fmt.Errorf("can't open %s: %v", localFile, err) } defer func() { if err := f.Close(); err != nil { @@ -259,21 +312,21 @@ func (b *Backuper) uploadSingleBackupFile(ctx context.Context, localFile, remote return b.dst.PutFile(ctx, remoteFile, f) }) if err != nil { - return fmt.Errorf("can't upload %s: %v", remoteFile, err) + return 0, fmt.Errorf("can't upload %s: %v", remoteFile, err) + } + info, err := os.Stat(localFile) + if err != nil { + return 0, fmt.Errorf("can't stat %s", localFile) } if b.resume { - info, err := os.Stat(localFile) - if err != nil { - return fmt.Errorf("can't stat %s", localFile) - } b.resumableState.AppendToState(remoteFile, info.Size()) } - return nil + return info.Size(), nil } func (b *Backuper) prepareTableListToUpload(ctx context.Context, backupName string, tablePattern string, partitions []string) (tablesForUpload ListOfTables, err error) { metadataPath := path.Join(b.DefaultDataPath, "backup", backupName, "metadata") - if b.isEmbedded { + if b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk != "" { metadataPath = path.Join(b.EmbeddedBackupDataPath, backupName, "metadata") } tablesForUpload, _, err = b.getTableListByPatternLocal(ctx, metadataPath, tablePattern, false, partitions) @@ -283,66 +336,6 @@ func (b *Backuper) prepareTableListToUpload(ctx context.Context, backupName stri return tablesForUpload, nil } -func (b *Backuper) getTablesForUploadDiffLocal(ctx context.Context, diffFrom string, backupMetadata *metadata.BackupMetadata, tablePattern string) (tablesForUploadFromDiff map[metadata.TableTitle]metadata.TableMetadata, err error) { - tablesForUploadFromDiff = make(map[metadata.TableTitle]metadata.TableMetadata) - diffFromBackup, err := b.ReadBackupMetadataLocal(ctx, diffFrom) - if err != nil { - return nil, err - } - if len(diffFromBackup.Tables) != 0 { - backupMetadata.RequiredBackup = diffFrom - metadataPath := path.Join(b.DefaultDataPath, "backup", diffFrom, "metadata") - // empty partitions, because we don't want filter - diffTablesList, _, err := b.getTableListByPatternLocal(ctx, metadataPath, tablePattern, false, []string{}) - if err != nil { - return nil, err - } - for _, t := range diffTablesList { - tablesForUploadFromDiff[metadata.TableTitle{ - Database: t.Database, - Table: t.Table, - }] = t - } - } - return tablesForUploadFromDiff, nil -} - -func (b *Backuper) getTablesForUploadDiffRemote(ctx context.Context, diffFromRemote string, backupMetadata *metadata.BackupMetadata, tablePattern string) (tablesForUploadFromDiff map[metadata.TableTitle]metadata.TableMetadata, err error) { - tablesForUploadFromDiff = make(map[metadata.TableTitle]metadata.TableMetadata) - backupList, err := b.dst.BackupList(ctx, true, diffFromRemote) - if err != nil { - return nil, err - } - var diffRemoteMetadata *metadata.BackupMetadata - for _, backup := range backupList { - if backup.BackupName == diffFromRemote { - if backup.Legacy { - return nil, fmt.Errorf("%s have legacy format and can't be used as diff-from-remote source", diffFromRemote) - } - diffRemoteMetadata = &backup.BackupMetadata - break - } - } - if diffRemoteMetadata == nil { - return nil, fmt.Errorf("%s not found on remote storage", diffFromRemote) - } - - if len(diffRemoteMetadata.Tables) != 0 { - backupMetadata.RequiredBackup = diffFromRemote - diffTablesList, err := getTableListByPatternRemote(ctx, b, diffRemoteMetadata, tablePattern, false) - if err != nil { - return nil, err - } - for _, t := range diffTablesList { - tablesForUploadFromDiff[metadata.TableTitle{ - Database: t.Database, - Table: t.Table, - }] = t - } - } - return tablesForUploadFromDiff, nil -} - func (b *Backuper) validateUploadParams(ctx context.Context, backupName string, diffFrom string, diffFromRemote string) error { if b.cfg.General.RemoteStorage == "none" { return fmt.Errorf("general->remote_storage shall not be \"none\" for upload, change you config or use REMOTE_STORAGE environment variable") @@ -366,31 +359,7 @@ func (b *Backuper) validateUploadParams(ctx context.Context, backupName string, if b.cfg.GetCompressionFormat() == "none" && !b.cfg.General.UploadByPart { return fmt.Errorf("%s->`compression_format`=%s incompatible with general->upload_by_part=%v", b.cfg.General.RemoteStorage, b.cfg.GetCompressionFormat(), b.cfg.General.UploadByPart) } - if (diffFrom != "" || diffFromRemote != "") && b.cfg.ClickHouse.UseEmbeddedBackupRestore { - log.Warn().Msgf("--diff-from and --diff-from-remote not compatible with backups created with `use_embedded_backup_restore: true`") - } - if b.cfg.ClickHouse.UseEmbeddedBackupRestore { - fatalMsg := fmt.Sprintf("`general->remote_storage: %s` `clickhouse->use_embedded_backup_restore: %v` require %s->compression_format: none, actual %%s", b.cfg.General.RemoteStorage, b.cfg.ClickHouse.UseEmbeddedBackupRestore, b.cfg.General.RemoteStorage) - if b.cfg.General.RemoteStorage == "s3" && b.cfg.S3.CompressionFormat != "none" { - log.Fatal().Stack().Msgf(fatalMsg, b.cfg.S3.CompressionFormat) - } - if b.cfg.General.RemoteStorage == "gcs" && b.cfg.GCS.CompressionFormat != "none" { - log.Fatal().Stack().Msgf(fatalMsg, b.cfg.GCS.CompressionFormat) - } - if b.cfg.General.RemoteStorage == "azblob" && b.cfg.AzureBlob.CompressionFormat != "none" { - log.Fatal().Stack().Msgf(fatalMsg, b.cfg.AzureBlob.CompressionFormat) - } - if b.cfg.General.RemoteStorage == "sftp" && b.cfg.SFTP.CompressionFormat != "none" { - log.Fatal().Stack().Msgf(fatalMsg, b.cfg.SFTP.CompressionFormat) - } - if b.cfg.General.RemoteStorage == "ftp" && b.cfg.FTP.CompressionFormat != "none" { - log.Fatal().Stack().Msgf(fatalMsg, b.cfg.FTP.CompressionFormat) - } - if b.cfg.General.RemoteStorage == "cos" && b.cfg.COS.CompressionFormat != "none" { - log.Fatal().Stack().Msgf(fatalMsg, b.cfg.COS.CompressionFormat) - } - } if b.cfg.General.RemoteStorage == "custom" && b.resume { return fmt.Errorf("can't resume for `remote_storage: custom`") } @@ -414,7 +383,12 @@ func (b *Backuper) validateUploadParams(ctx context.Context, backupName string, } func (b *Backuper) uploadConfigData(ctx context.Context, backupName string) (uint64, error) { - configBackupPath := path.Join(b.DefaultDataPath, "backup", backupName, "configs") + backupPath := b.DefaultDataPath + configBackupPath := path.Join(backupPath, "backup", backupName, "configs") + if b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk != "" { + backupPath = b.EmbeddedBackupDataPath + configBackupPath = path.Join(backupPath, backupName, "configs") + } configFilesGlobPattern := path.Join(configBackupPath, "**/*.*") if b.cfg.GetCompressionFormat() == "none" { remoteConfigsDir := path.Join(backupName, "configs") @@ -425,7 +399,12 @@ func (b *Backuper) uploadConfigData(ctx context.Context, backupName string) (uin } func (b *Backuper) uploadRBACData(ctx context.Context, backupName string) (uint64, error) { - rbacBackupPath := path.Join(b.DefaultDataPath, "backup", backupName, "access") + backupPath := b.DefaultDataPath + rbacBackupPath := path.Join(backupPath, "backup", backupName, "access") + if b.isEmbedded && b.cfg.ClickHouse.EmbeddedBackupDisk != "" { + backupPath = b.EmbeddedBackupDataPath + rbacBackupPath = path.Join(backupPath, backupName, "access") + } accessFilesGlobPattern := path.Join(rbacBackupPath, "*.*") if b.cfg.GetCompressionFormat() == "none" { remoteRBACDir := path.Join(backupName, "access") @@ -447,7 +426,11 @@ func (b *Backuper) uploadBackupRelatedDir(ctx context.Context, localBackupRelate var localFiles []string var err error if localFiles, err = filepathx.Glob(localFilesGlobPattern); err != nil || localFiles == nil || len(localFiles) == 0 { - return 0, fmt.Errorf("list %s return list=%v with err=%v", localFilesGlobPattern, localFiles, err) + if !b.cfg.General.RBACBackupAlways { + return 0, fmt.Errorf("list %s return list=%v with err=%v", localFilesGlobPattern, localFiles, err) + } + log.Warn().Msgf("list %s return list=%v with err=%v", localFilesGlobPattern, localFiles, err) + return 0, nil } for i := 0; i < len(localFiles); i++ { @@ -460,7 +443,7 @@ func (b *Backuper) uploadBackupRelatedDir(ctx context.Context, localBackupRelate } if b.cfg.GetCompressionFormat() == "none" { remoteUploadedBytes := int64(0) - if remoteUploadedBytes, err = b.dst.UploadPath(ctx, 0, localBackupRelatedDir, localFiles, destinationRemote, b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration); err != nil { + if remoteUploadedBytes, err = b.dst.UploadPath(ctx, localBackupRelatedDir, localFiles, destinationRemote, b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration, b.cfg.General.UploadMaxBytesPerSecond); err != nil { return 0, fmt.Errorf("can't RBAC or config upload %s: %v", destinationRemote, err) } if b.resume { @@ -470,12 +453,18 @@ func (b *Backuper) uploadBackupRelatedDir(ctx context.Context, localBackupRelate } retry := retrier.New(retrier.ConstantBackoff(b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration), nil) err = retry.RunCtx(ctx, func(ctx context.Context) error { - return b.dst.UploadCompressedStream(ctx, localBackupRelatedDir, localFiles, destinationRemote) + return b.dst.UploadCompressedStream(ctx, localBackupRelatedDir, localFiles, destinationRemote, b.cfg.General.UploadMaxBytesPerSecond) }) if err != nil { return 0, fmt.Errorf("can't RBAC or config upload compressed %s: %v", destinationRemote, err) } - remoteUploaded, err := b.dst.StatFile(ctx, destinationRemote) + + var remoteUploaded storage.RemoteFile + retry = retrier.New(retrier.ConstantBackoff(b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration), nil) + err = retry.RunCtx(ctx, func(ctx context.Context) error { + remoteUploaded, err = b.dst.StatFile(ctx, destinationRemote) + return err + }) if err != nil { return 0, fmt.Errorf("can't check uploaded destinationRemote: %s, error: %v", destinationRemote, err) } @@ -485,7 +474,7 @@ func (b *Backuper) uploadBackupRelatedDir(ctx context.Context, localBackupRelate return uint64(remoteUploaded.Size()), nil } -func (b *Backuper) uploadTableData(ctx context.Context, backupName string, table metadata.TableMetadata) (map[string][]string, int64, error) { +func (b *Backuper) uploadTableData(ctx context.Context, backupName string, deleteSource bool, table metadata.TableMetadata) (map[string][]string, int64, error) { dbAndTablePath := path.Join(common.TablePathEncode(table.Database), common.TablePathEncode(table.Table)) uploadedFiles := map[string][]string{} capacity := 0 @@ -493,8 +482,10 @@ func (b *Backuper) uploadTableData(ctx context.Context, backupName string, table capacity += len(table.Parts[disk]) } log.Debug().Msgf("start %s.%s with concurrency=%d len(table.Parts[...])=%d", table.Database, table.Table, b.cfg.General.UploadConcurrency, capacity) - s := semaphore.NewWeighted(int64(b.cfg.General.UploadConcurrency)) - g, ctx := errgroup.WithContext(ctx) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + dataGroup, ctx := errgroup.WithContext(ctx) + dataGroup.SetLimit(int(b.cfg.General.UploadConcurrency)) var uploadedBytes int64 splitParts := make(map[string][]metadata.SplitPartFiles) @@ -510,16 +501,11 @@ func (b *Backuper) uploadTableData(ctx context.Context, backupName string, table splitPartsOffset[disk] = 0 splitPartsCapacity += len(splitPartsList) } -breakByError: for common.SumMapValuesInt(splitPartsOffset) < splitPartsCapacity { for disk := range table.Parts { if splitPartsOffset[disk] >= len(splitParts[disk]) { continue } - if err := s.Acquire(ctx, 1); err != nil { - log.Error().Msgf("can't acquire semaphore during Upload data parts: %v", err) - break breakByError - } backupPath := b.getLocalBackupDataPathForTable(backupName, disk, dbAndTablePath) splitPart := splitParts[disk][splitPartsOffset[disk]] partSuffix := splitPart.Prefix @@ -529,8 +515,7 @@ breakByError: if b.cfg.GetCompressionFormat() == "none" { remotePath := path.Join(baseRemoteDataPath, disk) remotePathFull := path.Join(remotePath, partSuffix) - g.Go(func() error { - defer s.Release(1) + dataGroup.Go(func() error { if b.resume { if isProcessed, processedSize := b.resumableState.IsAlreadyProcessed(remotePathFull); isProcessed { atomic.AddInt64(&uploadedBytes, processedSize) @@ -538,7 +523,7 @@ breakByError: } } log.Debug().Msgf("start upload %d files to %s", len(partFiles), remotePath) - if uploadPathBytes, err := b.dst.UploadPath(ctx, 0, backupPath, partFiles, remotePath, b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration); err != nil { + if uploadPathBytes, err := b.dst.UploadPath(ctx, backupPath, partFiles, remotePath, b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration, b.cfg.General.UploadMaxBytesPerSecond); err != nil { log.Error().Msgf("UploadPath return error: %v", err) return fmt.Errorf("can't upload: %v", err) } else { @@ -547,7 +532,14 @@ breakByError: b.resumableState.AppendToState(remotePathFull, uploadPathBytes) } } - log.Debug().Msgf("finish upload %d files to %s", len(partFiles), remotePath) + // https://github.com/Altinity/clickhouse-backup/issues/777 + if deleteSource { + for _, f := range partFiles { + if err := os.Remove(path.Join(backupPath, f)); err != nil { + return fmt.Errorf("can't remove %s, %v", path.Join(backupPath, f), err) + } + } + } return nil }) } else { @@ -555,8 +547,7 @@ breakByError: uploadedFiles[disk] = append(uploadedFiles[disk], fileName) remoteDataFile := path.Join(baseRemoteDataPath, fileName) localFiles := partFiles - g.Go(func() error { - defer s.Release(1) + dataGroup.Go(func() error { if b.resume { if isProcessed, processedSize := b.resumableState.IsAlreadyProcessed(remoteDataFile); isProcessed { atomic.AddInt64(&uploadedBytes, processedSize) @@ -566,13 +557,19 @@ breakByError: log.Debug().Msgf("start upload %d files to %s", len(localFiles), remoteDataFile) retry := retrier.New(retrier.ConstantBackoff(b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration), nil) err := retry.RunCtx(ctx, func(ctx context.Context) error { - return b.dst.UploadCompressedStream(ctx, backupPath, localFiles, remoteDataFile) + return b.dst.UploadCompressedStream(ctx, backupPath, localFiles, remoteDataFile, b.cfg.General.UploadMaxBytesPerSecond) }) if err != nil { log.Error().Msgf("UploadCompressedStream return error: %v", err) return fmt.Errorf("can't upload: %v", err) } - remoteFile, err := b.dst.StatFile(ctx, remoteDataFile) + + var remoteFile storage.RemoteFile + retry = retrier.New(retrier.ConstantBackoff(b.cfg.General.RetriesOnFailure, b.cfg.General.RetriesDuration), nil) + err = retry.RunCtx(ctx, func(ctx context.Context) error { + remoteFile, err = b.dst.StatFile(ctx, remoteDataFile) + return err + }) if err != nil { return fmt.Errorf("can't check uploaded remoteDataFile: %s, error: %v", remoteDataFile, err) } @@ -580,22 +577,30 @@ breakByError: if b.resume { b.resumableState.AppendToState(remoteDataFile, remoteFile.Size()) } + // https://github.com/Altinity/clickhouse-backup/issues/777 + if deleteSource { + for _, f := range localFiles { + if err = os.Remove(path.Join(backupPath, f)); err != nil { + return fmt.Errorf("can't remove %s, %v", path.Join(backupPath, f), err) + } + } + } log.Debug().Msgf("finish upload to %s", remoteDataFile) return nil }) } } } - if err := g.Wait(); err != nil { + if err := dataGroup.Wait(); err != nil { return nil, 0, fmt.Errorf("one of uploadTableData go-routine return error: %v", err) } log.Debug().Msgf("finish %s.%s with concurrency=%d len(table.Parts[...])=%d uploadedFiles=%v, uploadedBytes=%v", table.Database, table.Table, b.cfg.General.UploadConcurrency, capacity, uploadedFiles, uploadedBytes) return uploadedFiles, uploadedBytes, nil } -func (b *Backuper) uploadTableMetadata(ctx context.Context, backupName string, tableMetadata metadata.TableMetadata) (int64, error) { +func (b *Backuper) uploadTableMetadata(ctx context.Context, backupName string, requiredBackupName string, tableMetadata metadata.TableMetadata) (int64, error) { if b.isEmbedded { - if sqlSize, err := b.uploadTableMetadataEmbedded(ctx, backupName, tableMetadata); err != nil { + if sqlSize, err := b.uploadTableMetadataEmbedded(ctx, backupName, requiredBackupName, tableMetadata); err != nil { return sqlSize, err } else { jsonSize, err := b.uploadTableMetadataRegular(ctx, backupName, tableMetadata) @@ -629,7 +634,10 @@ func (b *Backuper) uploadTableMetadataRegular(ctx context.Context, backupName st return int64(len(content)), nil } -func (b *Backuper) uploadTableMetadataEmbedded(ctx context.Context, backupName string, tableMetadata metadata.TableMetadata) (int64, error) { +func (b *Backuper) uploadTableMetadataEmbedded(ctx context.Context, backupName string, requiredBackupName string, tableMetadata metadata.TableMetadata) (int64, error) { + if b.cfg.ClickHouse.EmbeddedBackupDisk == "" { + return 0, nil + } remoteTableMetaFile := path.Join(backupName, "metadata", common.TablePathEncode(tableMetadata.Database), fmt.Sprintf("%s.sql", common.TablePathEncode(tableMetadata.Table))) if b.resume { if isProcessed, processedSize := b.resumableState.IsAlreadyProcessed(remoteTableMetaFile); isProcessed { @@ -637,9 +645,21 @@ func (b *Backuper) uploadTableMetadataEmbedded(ctx context.Context, backupName s } } localTableMetaFile := path.Join(b.EmbeddedBackupDataPath, backupName, "metadata", common.TablePathEncode(tableMetadata.Database), fmt.Sprintf("%s.sql", common.TablePathEncode(tableMetadata.Table))) - localReader, err := os.Open(localTableMetaFile) + var info os.FileInfo + var localReader *os.File + var err error + localReader, err = os.Open(localTableMetaFile) if err != nil { - return 0, fmt.Errorf("can't open %s: %v", localTableMetaFile, err) + err = fmt.Errorf("can't open %s: %v", localTableMetaFile, err) + if requiredBackupName != "" { + log.Warn().Err(err).Send() + return 0, nil + } else { + return 0, err + } + } + if info, err = os.Stat(localTableMetaFile); err != nil { + return 0, err } defer func() { if err := localReader.Close(); err != nil { @@ -653,14 +673,10 @@ func (b *Backuper) uploadTableMetadataEmbedded(ctx context.Context, backupName s if err != nil { return 0, fmt.Errorf("can't embeeded upload metadata: %v", err) } - if info, err := os.Stat(localTableMetaFile); err != nil { - return 0, fmt.Errorf("stat %s error: %v", localTableMetaFile, err) - } else { - if b.resume { - b.resumableState.AppendToState(remoteTableMetaFile, info.Size()) - } - return info.Size(), nil + if b.resume { + b.resumableState.AppendToState(remoteTableMetaFile, info.Size()) } + return info.Size(), nil } func (b *Backuper) markDuplicatedParts(backup *metadata.BackupMetadata, existsTable *metadata.TableMetadata, newTable *metadata.TableMetadata, checkLocal bool) { diff --git a/pkg/backup/utils.go b/pkg/backup/utils.go index d6c5ec5f..fee9aeae 100644 --- a/pkg/backup/utils.go +++ b/pkg/backup/utils.go @@ -4,7 +4,7 @@ import ( "sort" ) -func GetBackupsToDelete(backups []LocalBackup, keep int) []LocalBackup { +func GetBackupsToDeleteLocal(backups []LocalBackup, keep int) []LocalBackup { if len(backups) > keep { sort.SliceStable(backups, func(i, j int) bool { return backups[i].CreationDate.After(backups[j].CreationDate) diff --git a/pkg/backup/watch.go b/pkg/backup/watch.go index 52406cb8..2560f50e 100644 --- a/pkg/backup/watch.go +++ b/pkg/backup/watch.go @@ -3,15 +3,14 @@ package backup import ( "context" "fmt" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/server/metrics" + "github.com/Altinity/clickhouse-backup/v2/pkg/status" + "github.com/rs/zerolog/log" + "github.com/urfave/cli" "regexp" "strings" "time" - - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/server/metrics" - "github.com/Altinity/clickhouse-backup/pkg/status" - "github.com/rs/zerolog/log" - "github.com/urfave/cli" ) var watchBackupTemplateTimeRE = regexp.MustCompile(`{time:([^}]+)}`) @@ -83,6 +82,12 @@ func (b *Backuper) Watch(watchInterval, fullInterval, watchBackupNameTemplate, t prevBackupType := "" lastBackup := time.Now() lastFullBackup := time.Now() + + prevBackupName, prevBackupType, lastBackup, lastFullBackup, backupType, err = b.calculatePrevBackupNameAndType(ctx, prevBackupName, prevBackupType, lastBackup, lastFullBackup, backupType) + if err != nil { + return err + } + createRemoteErrCount := 0 deleteLocalErrCount := 0 var createRemoteErr error @@ -117,19 +122,39 @@ func (b *Backuper) Watch(watchInterval, fullInterval, watchBackupNameTemplate, t } if metrics != nil { createRemoteErr, createRemoteErrCount = metrics.ExecuteWithMetrics("create_remote", createRemoteErrCount, func() error { - return b.CreateToRemote(backupName, "", diffFromRemote, tablePattern, partitions, schemaOnly, backupRBAC, false, backupConfigs, false, skipCheckPartsColumns, false, version, commandId) + return b.CreateToRemote(backupName, false, "", diffFromRemote, tablePattern, partitions, schemaOnly, backupRBAC, false, backupConfigs, false, skipCheckPartsColumns, false, version, commandId) }) deleteLocalErr, deleteLocalErrCount = metrics.ExecuteWithMetrics("delete", deleteLocalErrCount, func() error { return b.RemoveBackupLocal(ctx, backupName, nil) }) } else { - createRemoteErr = b.CreateToRemote(backupName, "", diffFromRemote, tablePattern, partitions, schemaOnly, backupRBAC, false, backupConfigs, false, skipCheckPartsColumns, false, version, commandId) + createRemoteErr = b.CreateToRemote(backupName, false, "", diffFromRemote, tablePattern, partitions, schemaOnly, backupRBAC, false, backupConfigs, false, skipCheckPartsColumns, false, version, commandId) if createRemoteErr != nil { - log.Error().Fields(map[string]interface{}{ - "backup": backupName, - "operation": "watch", - }).Msgf("create_remote %s return error: %v", backupName, createRemoteErr) + cmd := "create_remote" + if diffFromRemote != "" { + cmd += " --diff-from-remote=" + diffFromRemote + } + if tablePattern != "" { + cmd += " --tables=" + tablePattern + } + if len(partitions) > 0 { + cmd += " --partition=" + strings.Join(partitions, ",") + } + if schemaOnly { + cmd += " --schema" + } + if backupRBAC { + cmd += " --rbac" + } + if backupConfigs { + cmd += " --configs" + } + if skipCheckPartsColumns { + cmd += " --skip-check-parts-columns" + } + cmd += " " + backupName + log.Error().Msgf("%s return error: %v", cmd, createRemoteErr) createRemoteErrCount += 1 } else { createRemoteErrCount = 0 @@ -180,3 +205,55 @@ func (b *Backuper) Watch(watchInterval, fullInterval, watchBackupNameTemplate, t } } } + +// calculatePrevBackupNameAndType - https://github.com/Altinity/clickhouse-backup/pull/804 +func (b *Backuper) calculatePrevBackupNameAndType(ctx context.Context, prevBackupName string, prevBackupType string, lastBackup time.Time, lastFullBackup time.Time, backupType string) (string, string, time.Time, time.Time, string, error) { + remoteBackups, err := b.GetRemoteBackups(ctx, true) + if err != nil { + return "", "", time.Time{}, time.Time{}, "", err + } + backupTemplateName, err := b.ch.ApplyMacros(ctx, b.cfg.General.WatchBackupNameTemplate) + if err != nil { + return "", "", time.Time{}, time.Time{}, "", err + } + backupTemplateNamePrepareRE := regexp.MustCompile(`{type}|{time:([^}]+)}`) + backupTemplateNameRE := regexp.MustCompile(backupTemplateNamePrepareRE.ReplaceAllString(backupTemplateName, `\S+`)) + + for _, remoteBackup := range remoteBackups { + if remoteBackup.Broken == "" && backupTemplateNameRE.MatchString(remoteBackup.BackupName) { + prevBackupName = remoteBackup.BackupName + if strings.Contains(remoteBackup.BackupName, "increment") { + prevBackupType = "increment" + lastBackup = remoteBackup.CreationDate + } else { + prevBackupType = "full" + lastBackup = remoteBackup.CreationDate + lastFullBackup = remoteBackup.CreationDate + } + } + } + if prevBackupName != "" { + now := time.Now() + timeBeforeDoBackup := int(b.cfg.General.WatchDuration.Seconds() - now.Sub(lastBackup).Seconds()) + timeBeforeDoFullBackup := int(b.cfg.General.FullDuration.Seconds() - now.Sub(lastFullBackup).Seconds()) + log.Info().Msgf("Time before do backup %v", timeBeforeDoBackup) + log.Info().Msgf("Time before do full backup %v", timeBeforeDoFullBackup) + if timeBeforeDoBackup > 0 && timeBeforeDoFullBackup > 0 { + log.Info().Msgf("Waiting %d seconds until continue doing backups due watch interval", timeBeforeDoBackup) + select { + case <-ctx.Done(): + return "", "", time.Time{}, time.Time{}, "", ctx.Err() + case <-time.After(b.cfg.General.WatchDuration - now.Sub(lastBackup)): + } + } + now = time.Now() + lastBackup = now + if b.cfg.General.FullDuration.Seconds()-time.Now().Sub(lastFullBackup).Seconds() <= 0 { + backupType = "full" + lastFullBackup = now + } else { + backupType = "increment" + } + } + return prevBackupName, prevBackupType, lastBackup, lastFullBackup, backupType, nil +} diff --git a/pkg/clickhouse/clickhouse.go b/pkg/clickhouse/clickhouse.go index aeea9a0a..e32fc434 100644 --- a/pkg/clickhouse/clickhouse.go +++ b/pkg/clickhouse/clickhouse.go @@ -15,21 +15,24 @@ import ( "strings" "time" - "github.com/Altinity/clickhouse-backup/pkg/common" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/metadata" "github.com/ClickHouse/clickhouse-go/v2" "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "github.com/antchfx/xmlquery" + "github.com/ricochet2200/go-disk-usage/du" + + "github.com/Altinity/clickhouse-backup/v2/pkg/common" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" + ) // ClickHouse - provide type ClickHouse struct { Config *config.ClickHouseConfig conn driver.Conn - disks []Disk version int isPartsColumnPresent int8 IsOpen bool @@ -56,12 +59,14 @@ func (ch *ClickHouse) Connect() error { Password: ch.Config.Password, }, Settings: clickhouse.Settings{ - // "connect_timeout": int(timeout.Seconds()), - // "receive_timeout": int(timeout.Seconds()), - // "send_timeout": int(timeout.Seconds()), + "connect_timeout": int(timeout.Seconds()), + "receive_timeout": int(timeout.Seconds()), + "send_timeout": int(timeout.Seconds()), + "http_send_timeout": 300, + "http_receive_timeout": 300, }, - MaxOpenConns: 1, - ConnMaxLifetime: 0, + MaxOpenConns: ch.Config.MaxConnections, + ConnMaxLifetime: 0, // don't change it, it related to SYSTEM SHUTDOWN behavior for properly rebuild RBAC lists on 20.4-22.3 MaxIdleConns: 0, DialTimeout: timeout, ReadTimeout: timeout, @@ -104,25 +109,32 @@ func (ch *ClickHouse) Connect() error { opt.Settings["log_queries"] = 0 } - if ch.conn, err = clickhouse.Open(opt); err != nil { - log.Error().Msgf("clickhouse connection: %s, clickhouse.Open return error: %v", fmt.Sprintf("tcp://%v:%v", ch.Config.Host, ch.Config.Port), err) - return err - } - logLevel := zerolog.InfoLevel if !ch.Config.LogSQLQueries { logLevel = zerolog.DebugLevel } - log.WithLevel(logLevel).Stack().Msgf("clickhouse connection prepared: %s run ping", fmt.Sprintf("tcp://%v:%v?timeout=%v", ch.Config.Host, ch.Config.Port, ch.Config.Timeout)) - err = ch.conn.Ping(context.Background()) - if err != nil { - log.Error().Msgf("clickhouse connection ping: %s return error: %v", fmt.Sprintf("tcp://%v:%v", ch.Config.Host, ch.Config.Port), err) - return err - } else { - ch.IsOpen = true + // infinite reconnect until success, fix https://github.com/Altinity/clickhouse-backup/issues/857 + for { + for { + ch.conn, err = clickhouse.Open(opt) + if err == nil { + break + } + log.Warn().Msgf("clickhouse connection: %s, sql.Open return error: %v, will wait 5 second to reconnect", fmt.Sprintf("tcp://%v:%v", ch.Config.Host, ch.Config.Port), err) + time.Sleep(5 * time.Second) + } + log.WithLevel(logLevel).Msgf("clickhouse connection prepared: %s run ping", fmt.Sprintf("tcp://%v:%v", ch.Config.Host, ch.Config.Port)) + err = ch.conn.Ping(context.Background()) + if err == nil { + log.WithLevel(logLevel).Msgf("clickhouse connection success: %s", fmt.Sprintf("tcp://%v:%v", ch.Config.Host, ch.Config.Port)) + ch.IsOpen = true + break + } + log.Warn().Msgf("clickhouse connection ping: %s return error: %v, will wait 5 second to reconnect", fmt.Sprintf("tcp://%v:%v", ch.Config.Host, ch.Config.Port), err) + time.Sleep(5 * time.Second) } - log.WithLevel(logLevel).Msgf("clickhouse connection open: %s", fmt.Sprintf("tcp://%v:%v", ch.Config.Host, ch.Config.Port)) - return err + + return nil } // GetDisks - return data from system.disks table @@ -181,12 +193,9 @@ func (ch *ClickHouse) GetDisks(ctx context.Context, enrich bool) ([]Disk, error) } func (ch *ClickHouse) GetEmbeddedBackupPath(disks []Disk) (string, error) { - if !ch.Config.UseEmbeddedBackupRestore { + if !ch.Config.UseEmbeddedBackupRestore || ch.Config.EmbeddedBackupDisk == "" { return "", nil } - if ch.Config.EmbeddedBackupDisk == "" { - return "", fmt.Errorf("please setup `clickhouse->embedded_backup_disk` in config or CLICKHOUSE_EMBEDDED_BACKUP_DISK environment variable") - } for _, d := range disks { if d.Name == ch.Config.EmbeddedBackupDisk { return d.Path, nil @@ -218,9 +227,11 @@ func (ch *ClickHouse) getDisksFromSystemSettings(ctx context.Context) ([]Disk, e dataPathArray := strings.Split(metadataPath, "/") clickhouseData := path.Join(dataPathArray[:len(dataPathArray)-1]...) return []Disk{{ - Name: "default", - Path: path.Join("/", clickhouseData), - Type: "local", + Name: "default", + Path: path.Join("/", clickhouseData), + Type: "local", + FreeSpace: du.NewDiskUsage(path.Join("/", clickhouseData)).Free(), + StoragePolicies: []string{"default"}, }}, nil } } @@ -251,18 +262,48 @@ func (ch *ClickHouse) getDisksFromSystemDisks(ctx context.Context) ([]Disk, erro case <-ctx.Done(): return nil, ctx.Err() default: - isDiskType := make([]struct { - Present uint64 `ch:"is_disk_type_present"` - }, 0) - if err := ch.SelectContext(ctx, &isDiskType, "SELECT count() is_disk_type_present FROM system.columns WHERE database='system' AND table='disks' AND name='type'"); err != nil { + type DiskFields struct { + DiskTypePresent uint64 `ch:"is_disk_type_present"` + ObjectStorageTypePresent uint64 `ch:"is_object_storage_type_present"` + FreeSpacePresent uint64 `ch:"is_free_space_present"` + StoragePolicyPresent uint64 `ch:"is_storage_policy_present"` + } + diskFields := make([]DiskFields, 0) + if err := ch.SelectContext(ctx, &diskFields, + "SELECT countIf(name='type') AS is_disk_type_present, "+ + "countIf(name='object_storage_type') AS is_object_storage_type_present, "+ + "countIf(name='free_space') AS is_free_space_present, "+ + "countIf(name='disks') AS is_storage_policy_present "+ + "FROM system.columns WHERE database='system' AND table IN ('disks','storage_policies') ", + ); err != nil { return nil, err } diskTypeSQL := "'local'" - if len(isDiskType) > 0 && isDiskType[0].Present > 0 { - diskTypeSQL = "any(type)" + if len(diskFields) > 0 && diskFields[0].DiskTypePresent > 0 { + diskTypeSQL = "any(d.type)" + } + if len(diskFields) > 0 && diskFields[0].ObjectStorageTypePresent > 0 { + diskTypeSQL = "any(lower(if(d.type='ObjectStorage',d.object_storage_type,d.type)))" + } + + diskFreeSpaceSQL := "toUInt64(0)" + if len(diskFields) > 0 && diskFields[0].FreeSpacePresent > 0 { + diskFreeSpaceSQL = "min(d.free_space)" + } + storagePoliciesSQL := "['default']" + joinStoragePoliciesSQL := "" + if len(diskFields) > 0 && diskFields[0].StoragePolicyPresent > 0 { + storagePoliciesSQL = "groupUniqArray(s.policy_name)" + // LEFT JOIN to allow disks which not have policy, https://github.com/Altinity/clickhouse-backup/issues/845 + joinStoragePoliciesSQL = " LEFT JOIN " + joinStoragePoliciesSQL += "(SELECT policy_name, arrayJoin(disks) AS disk FROM system.storage_policies) AS s ON s.disk = d.name" } var result []Disk - query := fmt.Sprintf("SELECT path, any(name) AS name, %s AS type FROM system.disks GROUP BY path", diskTypeSQL) + query := fmt.Sprintf( + "SELECT d.path, any(d.name) AS name, %s AS type, %s AS free_space, %s AS storage_policies "+ + "FROM system.disks AS d %s GROUP BY d.path", + diskTypeSQL, diskFreeSpaceSQL, storagePoliciesSQL, joinStoragePoliciesSQL, + ) err := ch.SelectContext(ctx, &result, query) return result, err } @@ -296,7 +337,7 @@ func (ch *ClickHouse) GetTables(ctx context.Context, tablePattern string) ([]Tab skipDatabases := make([]struct { Name string `ch:"name"` }, 0) - // MaterializedPostgreSQL doesn't support FREEZE look https://github.com/Altinity/clickhouse-backup/issues/550 + // MaterializedPostgreSQL doesn't support FREEZE look https://github.com/Altinity/clickhouse-backup/issues/550 and https://github.com/ClickHouse/ClickHouse/issues/32902 if err = ch.SelectContext(ctx, &skipDatabases, "SELECT name FROM system.databases WHERE engine IN ('MySQL','PostgreSQL','MaterializedPostgreSQL')"); err != nil { return nil, err } @@ -340,6 +381,12 @@ func (ch *ClickHouse) GetTables(ctx context.Context, tablePattern string) ([]Tab if ch.Config.UseEmbeddedBackupRestore && (strings.HasPrefix(t.Name, ".inner_id.") /*|| strings.HasPrefix(t.Name, ".inner.")*/) { t.Skip = true } + for _, engine := range ch.Config.SkipTableEngines { + if t.Engine == engine { + t.Skip = true + break + } + } if t.Skip { tables[i] = t continue @@ -429,14 +476,14 @@ func (ch *ClickHouse) prepareGetTablesSQL(tablePattern string, skipDatabases, sk allTablesSQL += " FROM system.tables WHERE is_temporary = 0" if tablePattern != "" { - replacer := strings.NewReplacer(".", "\\.", ",", "$|^", "*", ".*", "?", ".", " ", "", "`", "", `"`, "", "-", "\\-") + replacer := strings.NewReplacer(".", "\\.", "$", ".", ",", "$|^", "*", ".*", "?", ".", " ", "", "`", "", `"`, "", "-", "\\-") allTablesSQL += fmt.Sprintf(" AND match(concat(database,'.',name),'^%s$') ", replacer.Replace(tablePattern)) } if len(skipDatabases) > 0 { allTablesSQL += fmt.Sprintf(" AND database NOT IN ('%s')", strings.Join(skipDatabases, "','")) } if len(skipTableEngines) > 0 { - allTablesSQL += fmt.Sprintf(" AND engine NOT IN ('%s')", strings.Join(skipTableEngines, "','")) + allTablesSQL += fmt.Sprintf(" AND NOT has(arrayMap(x->lower(x), ['%s']), lower(engine))", strings.Join(skipTableEngines, "','")) } // try to upload big tables first if len(isSystemTablesFieldPresent) > 0 && isSystemTablesFieldPresent[0].IsTotalBytesPresent > 0 { @@ -620,16 +667,16 @@ func (ch *ClickHouse) GetVersion(ctx context.Context) (int, error) { func (ch *ClickHouse) GetVersionDescribe(ctx context.Context) string { var result string - query := "SELECT value FROM `system`.`build_options` where name='VERSION_DESCRIBE'" + query := "SELECT value FROM `system`.`build_options` WHERE name='VERSION_DESCRIBE'" if err := ch.SelectSingleRow(ctx, &result, query); err != nil { return "" } return result } -// FreezeTableOldWay - freeze all partitions in table one by one -// This way using for ClickHouse below v19.1 -func (ch *ClickHouse) FreezeTableOldWay(ctx context.Context, table *Table, name string) error { +// FreezeTableByParts - freeze all partitions in table one by one +// also ally `freeze_by_part_where` +func (ch *ClickHouse) FreezeTableByParts(ctx context.Context, table *Table, name string) error { var partitions []struct { PartitionID string `ch:"partition_id"` } @@ -685,7 +732,7 @@ func (ch *ClickHouse) FreezeTable(ctx context.Context, table *Table, name string } } if version < 19001005 || ch.Config.FreezeByPart { - return ch.FreezeTableOldWay(ctx, table, name) + return ch.FreezeTableByParts(ctx, table, name) } withNameQuery := "" if name != "" { @@ -703,7 +750,13 @@ func (ch *ClickHouse) FreezeTable(ctx context.Context, table *Table, name string } // AttachDataParts - execute ALTER TABLE ... ATTACH PART command for specific table -func (ch *ClickHouse) AttachDataParts(table metadata.TableMetadata, disks []Disk) error { +func (ch *ClickHouse) AttachDataParts(table metadata.TableMetadata, dstTable Table) error { + if dstTable.Database != "" && dstTable.Database != table.Database { + table.Database = dstTable.Database + } + if dstTable.Name != "" && dstTable.Name != table.Table { + table.Table = dstTable.Name + } canContinue, err := ch.CheckReplicationInProgress(table) if err != nil { return err @@ -711,14 +764,14 @@ func (ch *ClickHouse) AttachDataParts(table metadata.TableMetadata, disks []Disk if !canContinue { return nil } - for _, disk := range disks { - for _, part := range table.Parts[disk.Name] { + for disk := range table.Parts { + for _, part := range table.Parts[disk] { if !strings.HasSuffix(part.Name, ".proj") { query := fmt.Sprintf("ALTER TABLE `%s`.`%s` ATTACH PART '%s'", table.Database, table.Table, part.Name) if err := ch.Query(query); err != nil { return err } - log.Debug().Str("table", fmt.Sprintf("%s.%s", table.Database, table.Table)).Str("disk", disk.Name).Str("part", part.Name).Msg("attached") + log.Debug().Str("table", fmt.Sprintf("%s.%s", table.Database, table.Table)).Str("disk", disk).Str("part", part.Name).Msg("attached") } } } @@ -729,11 +782,17 @@ var replicatedMergeTreeRE = regexp.MustCompile(`Replicated[\w_]*MergeTree\s*\((( var uuidRE = regexp.MustCompile(`UUID '([^']+)'`) // AttachTable - execute ATTACH TABLE command for specific table -func (ch *ClickHouse) AttachTable(ctx context.Context, table metadata.TableMetadata) error { +func (ch *ClickHouse) AttachTable(ctx context.Context, table metadata.TableMetadata, dstTable Table) error { if len(table.Parts) == 0 { log.Warn().Msgf("no data parts for restore for `%s`.`%s`", table.Database, table.Table) return nil } + if dstTable.Database != "" && dstTable.Database != table.Database { + table.Database = dstTable.Database + } + if dstTable.Name != "" && dstTable.Name != table.Table { + table.Table = dstTable.Name + } canContinue, err := ch.CheckReplicationInProgress(table) if err != nil { return err @@ -742,13 +801,17 @@ func (ch *ClickHouse) AttachTable(ctx context.Context, table metadata.TableMetad return nil } - query := fmt.Sprintf("DETACH TABLE `%s`.`%s`", table.Database, table.Table) + if ch.version <= 21003000 { + return fmt.Errorf("your clickhouse-server version doesn't support SYSTEM RESTORE REPLICA statement, use `restore_as_attach: false` in config") + } + query := fmt.Sprintf("DETACH TABLE `%s`.`%s` SYNC", table.Database, table.Table) if err := ch.Query(query); err != nil { return err } - if matches := replicatedMergeTreeRE.FindStringSubmatch(table.Query); len(matches) > 0 { - zkPath := strings.Trim(matches[2], "' \r\n\t") - replicaName := strings.Trim(matches[3], "' \r\n\t") + replicatedMatches := replicatedMergeTreeRE.FindStringSubmatch(table.Query) + if len(replicatedMatches) > 0 { + zkPath := strings.Trim(replicatedMatches[2], "' \r\n\t") + replicaName := strings.Trim(replicatedMatches[3], "' \r\n\t") if strings.Contains(zkPath, "{uuid}") { if uuidMatches := uuidRE.FindStringSubmatch(table.Query); len(uuidMatches) > 0 { zkPath = strings.Replace(zkPath, "{uuid}", uuidMatches[1], 1) @@ -773,11 +836,12 @@ func (ch *ClickHouse) AttachTable(ctx context.Context, table metadata.TableMetad return err } - query = fmt.Sprintf("SYSTEM RESTORE REPLICA `%s`.`%s`", table.Database, table.Table) - if err := ch.Query(query); err != nil { - return err + if len(replicatedMatches) > 0 { + query = fmt.Sprintf("SYSTEM RESTORE REPLICA `%s`.`%s`", table.Database, table.Table) + if err := ch.Query(query); err != nil { + return err + } } - log.Debug().Str("table", fmt.Sprintf("%s.%s", table.Database, table.Table)).Msg("attached") return nil } @@ -827,7 +891,7 @@ func (ch *ClickHouse) addOnClusterToCreateDatabase(cluster string, query string) } // DropTable - drop ClickHouse table -func (ch *ClickHouse) DropTable(table Table, query string, onCluster string, ignoreDependencies bool, version int) error { +func (ch *ClickHouse) DropTable(table Table, query string, onCluster string, ignoreDependencies bool, version int, defaultDataPath string) error { var isAtomic bool var err error if isAtomic, err = ch.IsAtomic(table.Database); err != nil { @@ -847,6 +911,11 @@ func (ch *ClickHouse) DropTable(table Table, query string, onCluster string, ign if ignoreDependencies { dropQuery += " SETTINGS check_table_dependencies=0" } + if defaultDataPath != "" { + if _, err = os.Create(path.Join(defaultDataPath, "/flags/force_drop_table")); err != nil { + return err + } + } if err = ch.Query(dropQuery); err != nil { return err } @@ -862,10 +931,10 @@ var onClusterRe = regexp.MustCompile(`(?im)\s+ON\s+CLUSTER\s+`) var distributedRE = regexp.MustCompile(`(Distributed)\(([^,]+),([^)]+)\)`) // CreateTable - create ClickHouse table -func (ch *ClickHouse) CreateTable(table Table, query string, dropTable, ignoreDependencies bool, onCluster string, version int) error { +func (ch *ClickHouse) CreateTable(table Table, query string, dropTable, ignoreDependencies bool, onCluster string, version int, defaultDataPath string) error { var err error if dropTable { - if err = ch.DropTable(table, query, onCluster, ignoreDependencies, version); err != nil { + if err = ch.DropTable(table, query, onCluster, ignoreDependencies, version, defaultDataPath); err != nil { return err } } @@ -884,7 +953,7 @@ func (ch *ClickHouse) CreateTable(table Table, query string, dropTable, ignoreDe return errors.New(fmt.Sprintf("schema query ```%s``` doesn't contains table name `%s`", query, table.Name)) } - // fix restore schema for legacy backup + // fix schema for restore // see https://github.com/Altinity/clickhouse-backup/issues/268 // https://github.com/Altinity/clickhouse-backup/issues/297 // https://github.com/Altinity/clickhouse-backup/issues/331 @@ -914,40 +983,50 @@ func (ch *ClickHouse) CreateTable(table Table, query string, dropTable, ignoreDe } } + // WINDOW VIEW unavailable after 24.3 + allowExperimentalAnalyzer := "" + if allowExperimentalAnalyzer, err = ch.TurnAnalyzerOffIfNecessary(version, query, allowExperimentalAnalyzer); err != nil { + return err + } + if err := ch.Query(query); err != nil { return err } - return nil -} -// GetConn - return current connection -func (ch *ClickHouse) GetConn() driver.Conn { - return ch.conn + // WINDOW VIEW unavailable after 24.3 + if err = ch.TurnAnalyzerOnIfNecessary(version, query, allowExperimentalAnalyzer); err != nil { + return err + } + return nil } -func (ch *ClickHouse) IsClickhouseShadow(path string) bool { - d, err := os.Open(path) - if err != nil { - return false - } - defer func() { - if err := d.Close(); err != nil { - log.Warn().Msgf("can't close directory %v", err) +func (ch *ClickHouse) TurnAnalyzerOnIfNecessary(version int, query string, allowExperimentalAnalyzer string) error { + if version > 24003000 && (strings.HasPrefix(query, "CREATE LIVE VIEW") || strings.HasPrefix(query, "ATTACH LIVE VIEW") || strings.HasPrefix(query, "CREATE WINDOW VIEW") || strings.HasPrefix(query, "ATTACH WINDOW VIEW")) && allowExperimentalAnalyzer == "1" { + if err := ch.Query("SET allow_experimental_analyzer=1"); err != nil { + return err } - }() - names, err := d.Readdirnames(-1) - if err != nil { - return false } - for _, name := range names { - if name == "increment.txt" { - continue + return nil +} + +func (ch *ClickHouse) TurnAnalyzerOffIfNecessary(version int, query string, allowExperimentalAnalyzer string) (string, error) { + if version > 24003000 && (strings.HasPrefix(query, "CREATE LIVE VIEW") || strings.HasPrefix(query, "ATTACH LIVE VIEW") || strings.HasPrefix(query, "CREATE WINDOW VIEW") || strings.HasPrefix(query, "ATTACH WINDOW VIEW")) { + if err := ch.SelectSingleRowNoCtx(&allowExperimentalAnalyzer, "SELECT value FROM system.settings WHERE name='allow_experimental_analyzer'"); err != nil { + return "", err } - if _, err := strconv.Atoi(name); err != nil { - return false + if allowExperimentalAnalyzer == "1" { + if err := ch.Query("SET allow_experimental_analyzer=0"); err != nil { + return "", err + } } + return allowExperimentalAnalyzer, nil } - return true + return "", nil +} + +// GetConn - return current connection +func (ch *ClickHouse) GetConn() driver.Conn { + return ch.conn } func (ch *ClickHouse) StructSelect(dest interface{}, query string, args ...interface{}) error { @@ -1078,6 +1157,7 @@ func (ch *ClickHouse) CalculateMaxFileSize(ctx context.Context, cfg *config.Conf if !cfg.General.UploadByPart { maxSizeQuery = "SELECT toInt64(max(data_by_disk) * 1.02) AS max_file_size FROM (SELECT disk_name, max(toInt64(bytes_on_disk)) data_by_disk FROM system.parts GROUP BY disk_name)" } + maxSizeQuery += " SETTINGS empty_result_for_aggregation_by_empty_set=0" if err := ch.SelectSingleRow(ctx, &rows, maxSizeQuery); err != nil { return 0, fmt.Errorf("can't calculate max(bytes_on_disk): %v", err) } @@ -1127,17 +1207,22 @@ func (ch *ClickHouse) ApplyMutation(ctx context.Context, tableMetadata metadata. func (ch *ClickHouse) CheckReplicationInProgress(table metadata.TableMetadata) (bool, error) { if ch.Config.CheckReplicasBeforeAttach && strings.Contains(table.Query, "Replicated") { existsReplicas := make([]struct { - InProgress uint64 `ch:"replication_in_progress"` + LogPointer uint64 `ch:"log_pointer"` + LogMaxIndex uint64 `ch:"log_max_index"` + AbsoluteDelay uint64 `ch:"absolute_delay"` + QueueSize uint32 `ch:"queue_size"` }, 0) - if err := ch.Select(&existsReplicas, "SELECT sum(log_pointer + log_max_index + absolute_delay + queue_size) AS replication_in_progress FROM system.replicas WHERE database=? and table=? SETTINGS empty_result_for_aggregation_by_empty_set=0", table.Database, table.Table); err != nil { + if err := ch.Select(&existsReplicas, "SELECT log_pointer, log_max_index, absolute_delay, queue_size FROM system.replicas WHERE database=? and table=?", table.Database, table.Table); err != nil { return false, err } - if len(existsReplicas) != 1 { + if len(existsReplicas) == 0 { + return true, nil + } + if len(existsReplicas) > 1 { return false, fmt.Errorf("invalid result for check exists replicas: %+v", existsReplicas) } - if existsReplicas[0].InProgress > 0 { - log.Warn().Msgf("%s.%s skipped cause system.replicas entry already exists and replication in progress from another replica", table.Database, table.Table) - return false, nil + if existsReplicas[0].LogPointer > 1 || existsReplicas[0].LogMaxIndex > 1 || existsReplicas[0].AbsoluteDelay > 0 || existsReplicas[0].QueueSize > 0 { + return false, fmt.Errorf("%s.%s can't restore cause system.replicas entries already exists and replication in progress from another replica, log_pointer=%d, log_max_index=%d, absolute_delay=%d, queue_size=%d", table.Database, table.Table, existsReplicas[0].LogPointer, existsReplicas[0].LogMaxIndex, existsReplicas[0].AbsoluteDelay, existsReplicas[0].QueueSize) } else { log.Info().Msgf("replication_in_progress status = %+v", existsReplicas) } @@ -1147,51 +1232,49 @@ func (ch *ClickHouse) CheckReplicationInProgress(table metadata.TableMetadata) ( // CheckSystemPartsColumns check data parts types consistency https://github.com/Altinity/clickhouse-backup/issues/529#issuecomment-1554460504 func (ch *ClickHouse) CheckSystemPartsColumns(ctx context.Context, table *Table) error { - if ch.isPartsColumnPresent == -1 { - return nil - } - if ch.isPartsColumnPresent == 0 { - isPartsColumn := make([]struct { - Present uint64 `ch:"is_parts_column_present"` - }, 0) - if err := ch.SelectContext(ctx, &isPartsColumn, "SELECT count() is_parts_column_present FROM system.tables WHERE database='system' AND name='parts_columns'"); err != nil { - return err - } - if len(isPartsColumn) != 1 || isPartsColumn[0].Present != 1 { - ch.isPartsColumnPresent = -1 - return nil - } - } - ch.isPartsColumnPresent = 1 - partColumnsDataTypes := make([]struct { - Column string `ch:"column"` - Types []string `ch:"uniq_types"` - }, 0) + var err error + partColumnsDataTypes := make([]ColumnDataTypes, 0) partsColumnsSQL := "SELECT column, groupUniqArray(type) AS uniq_types " + "FROM system.parts_columns " + - "WHERE active AND database=? AND table=? AND type NOT LIKE 'Enum%' AND type NOT LIKE 'Tuple(%' " + + "WHERE active AND database=? AND table=? AND type NOT LIKE 'Enum%(%' AND type NOT LIKE 'Tuple(%' AND type NOT LIKE 'Array(Tuple(%' " + "GROUP BY column HAVING length(uniq_types) > 1" - if err := ch.SelectContext(ctx, &partColumnsDataTypes, partsColumnsSQL, table.Database, table.Name); err != nil { + if err = ch.SelectContext(ctx, &partColumnsDataTypes, partsColumnsSQL, table.Database, table.Name); err != nil { return err } - isPartColumnsInconsistentDataTypes := false - if len(partColumnsDataTypes) > 0 { - for i := range partColumnsDataTypes { - isNullablePresent := false - isNotNullablePresent := false - for _, dataType := range partColumnsDataTypes[i].Types { - if strings.Contains(dataType, "Nullable") { - isNullablePresent = true - } else { - isNotNullablePresent = true - } + return ch.CheckTypesConsistency(table, partColumnsDataTypes) +} + +var dateWithParams = regexp.MustCompile(`^(Date[^(]+)\([^)]+\)`) +var versioningAggregateRE = regexp.MustCompile(`^[0-9]+,\s*`) + +func (ch *ClickHouse) CheckTypesConsistency(table *Table, partColumnsDataTypes []ColumnDataTypes) error { + cleanType := func(dataType string) string { + for _, compatiblePrefix := range []string{"LowCardinality(", "Nullable("} { + if strings.HasPrefix(dataType, compatiblePrefix) { + dataType = strings.TrimPrefix(dataType, compatiblePrefix) + dataType = strings.TrimSuffix(dataType, ")") } - if !isNullablePresent && isNotNullablePresent { - log.Error().Msgf("`%s`.`%s` have inconsistent data types %#v for \"%s\" column", table.Database, table.Name, partColumnsDataTypes[i].Types, partColumnsDataTypes[i].Column) - isPartColumnsInconsistentDataTypes = true + } + dataType = dateWithParams.ReplaceAllString(dataType, "$1") + return dataType + } + for i := range partColumnsDataTypes { + isAggregationPresent := false + uniqTypes := common.EmptyMap{} + for _, dataType := range partColumnsDataTypes[i].Types { + isAggregationPresent = strings.Contains(dataType, "AggregateFunction(") + if isAggregationPresent { + dataType = strings.TrimPrefix(dataType, "SimpleAggregateFunction(") + dataType = strings.TrimPrefix(dataType, "AggregateFunction(") + dataType = strings.TrimSuffix(dataType, ")") + dataType = versioningAggregateRE.ReplaceAllString(dataType, "") + } else { + dataType = cleanType(dataType) } + uniqTypes[dataType] = struct{}{} } - if isPartColumnsInconsistentDataTypes { + if len(uniqTypes) > 1 { + log.Error().Msgf("`%s`.`%s` have incompatible data types %#v for \"%s\" column", table.Database, table.Name, partColumnsDataTypes[i].Types, partColumnsDataTypes[i].Column) return fmt.Errorf("`%s`.`%s` have inconsistent data types for active data part in system.parts_columns", table.Database, table.Name) } } @@ -1279,3 +1362,15 @@ func (ch *ClickHouse) GetPreprocessedXMLSettings(ctx context.Context, settingsXP } return resultSettings, nil } + +var storagePolicyRE = regexp.MustCompile(`SETTINGS.+storage_policy[^=]*=[^']*'([^']+)'`) + +func (ch *ClickHouse) ExtractStoragePolicy(query string) string { + storagePolicy := "default" + matches := storagePolicyRE.FindStringSubmatch(query) + if len(matches) > 0 { + storagePolicy = matches[1] + } + log.Debug().Msgf("extract storage_policy: %s, query: %s", storagePolicy, query) + return storagePolicy +} diff --git a/pkg/clickhouse/clickhouse_test.go b/pkg/clickhouse/clickhouse_test.go new file mode 100644 index 00000000..ac24f334 --- /dev/null +++ b/pkg/clickhouse/clickhouse_test.go @@ -0,0 +1,120 @@ +package clickhouse + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCheckTypesConsistency(t *testing.T) { + ch := ClickHouse{} + table := &Table{ + Database: "mydb", + Name: "mytable", + } + expectedErr := fmt.Errorf("`mydb`.`mytable` have inconsistent data types for active data part in system.parts_columns") + + testCases := []struct { + Name string + PartColumnsData []ColumnDataTypes + ExpectedError error + }{ + { + Name: "No partColumnsData", + PartColumnsData: []ColumnDataTypes{}, + ExpectedError: nil, + }, + { + Name: "Consistent data types", + PartColumnsData: []ColumnDataTypes{ + { + Column: "agg_col", + Types: []string{"AggregateFunction(1, sumMap, Array(UInt16), Array(UInt64))", "AggregateFunction(sumMap, Array(UInt16), Array(UInt64))"}, + }, + { + Column: "simple_agg_col", + Types: []string{"SimpleAggregateFunction(1, sum, UInt16)", "SimpleAggregateFunction(sum, UInt16)"}, + }, + + { + Column: "col3", + Types: []string{"Nullable(Int32)", "Int32"}, + }, + { + Column: "col4", + Types: []string{"LowCardinality(String)", "String"}, + }, + { + Column: "col5", + Types: []string{"DateTime", "DateTime('Meteor/Chelyabinsk')"}, + }, + { + Column: "col6", + Types: []string{"LowCardinality(Nullable(String))", "String"}, + }, + }, + ExpectedError: nil, + }, + { + Name: "Inconsistent data types", + PartColumnsData: []ColumnDataTypes{ + { + Column: "col1", + Types: []string{"Int32", "String"}, + }, + }, + ExpectedError: expectedErr, + }, + { + Name: "Inconsistent AggregateFunction", + PartColumnsData: []ColumnDataTypes{ + { + Column: "agg_col", + Types: []string{"AggregateFunction(1, avg, Array(UInt16), Array(UInt64))", "AggregateFunction(sumMap, Array(UInt16), Array(UInt64))"}, + }, + }, + ExpectedError: expectedErr, + }, + { + Name: "Inconsistent SimpleAggregateFunction", + PartColumnsData: []ColumnDataTypes{ + { + Column: "simple_agg_col", + Types: []string{"SimpleAggregateFunction(1, sum, UInt16)", "SimpleAggregateFunction(sumMap, Array(UInt16))"}, + }, + }, + ExpectedError: expectedErr, + }, + { + Name: "Inconsistent Types #2", + PartColumnsData: []ColumnDataTypes{ + { + Column: "col2", + Types: []string{"DateTime(6)", "Date"}, + }, + }, + ExpectedError: expectedErr, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + err := ch.CheckTypesConsistency(table, tc.PartColumnsData) + assert.Equal(t, tc.ExpectedError, err) + }) + } +} + +func TestExtractStoragePolicy(t *testing.T) { + ch := ClickHouse{} + + testCases := map[string]string{ + "CREATE TABLE `_test.ДБ_atomic__TestIntegrationS3`.test_s3_TestIntegrationS3 UUID '8135780b-0c9a-46a7-94fd-2aebb701eff6' (`id` UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/_test.ДБ_atomic__TestIntegrationS3/test_s3_TestIntegrationS3', '{replica}') ORDER BY id SETTINGS storage_policy = 's3_only', index_granularity = 8192": "s3_only", + "CREATE TABLE test2 SETTINGS storage_policy = 'default'": "default", + "CREATE TABLE test3": "default", + } + for query, policy := range testCases { + assert.Equal(t, policy, ch.ExtractStoragePolicy(query)) + } +} diff --git a/pkg/clickhouse/legacy.go b/pkg/clickhouse/legacy.go deleted file mode 100644 index 767278a5..00000000 --- a/pkg/clickhouse/legacy.go +++ /dev/null @@ -1,94 +0,0 @@ -package clickhouse - -import ( - "fmt" - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/Altinity/clickhouse-backup/pkg/metadata" -) - -// GetBackupTablesLegacy - return list of backups of tables that can be restored -func (ch *ClickHouse) GetBackupTablesLegacy(backupName string, disks []Disk) ([]metadata.TableMetadata, error) { - dataPath, err := ch.GetDefaultPath(disks) - if err != nil { - return nil, err - } - backupShadowPath := filepath.Join(dataPath, "backup", backupName, "shadow") - dbNum := 0 - tableNum := 1 - partNum := 2 - totalNum := 3 - if ch.IsClickhouseShadow(backupShadowPath) { - dbNum = 2 - tableNum = 3 - partNum = 4 - totalNum = 5 - } - fi, err := os.Stat(backupShadowPath) - if err != nil { - return nil, fmt.Errorf("can't get tables, %v", err) - } - if !fi.IsDir() { - return nil, fmt.Errorf("can't get tables, %s is not a dir", backupShadowPath) - } - - var allpartsBackup map[string][]metadata.Part - // TODO: we don't need anymore fill Partitions from file, we can get it from `system.detached_parts` table - tables := make(map[string]metadata.TableMetadata) - err = filepath.Walk(backupShadowPath, func(filePath string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - filePath = filepath.ToSlash(filePath) // fix fucking Windows slashes - relativePath := strings.Trim(strings.TrimPrefix(filePath, backupShadowPath), "/") - parts := strings.Split(relativePath, "/") - if len(parts) != totalNum { - return nil - } - - tDB, _ := url.PathUnescape(parts[dbNum]) - tName, _ := url.PathUnescape(parts[tableNum]) - fullTableName := fmt.Sprintf("%s.%s", tDB, tName) - - allPartsHashes := allpartsBackup[fullTableName] - var hashOfAllFiles, hashOfUncompressedFiles, uncompressHashOfCompressedFiles string - for _, partHash := range allPartsHashes { - if partHash.Name == parts[partNum] { - hashOfAllFiles = partHash.HashOfAllFiles - hashOfUncompressedFiles = partHash.HashOfUncompressedFiles - uncompressHashOfCompressedFiles = partHash.UncompressedHashOfCompressedFiles - } - } - - partition := metadata.Part{ - Name: parts[partNum], - // Path: filePath, - HashOfAllFiles: hashOfAllFiles, - HashOfUncompressedFiles: hashOfUncompressedFiles, - UncompressedHashOfCompressedFiles: uncompressHashOfCompressedFiles, - } - - if t, ok := tables[fullTableName]; ok { - t.Parts["default"] = append(t.Parts["default"], partition) - tables[fullTableName] = t - return nil - } - tables[fullTableName] = metadata.TableMetadata{ - Database: tDB, - Table: tName, - Parts: map[string][]metadata.Part{"default": {partition}}, - } - return nil - } - return nil - }) - result := make([]metadata.TableMetadata, 0) - for i := range tables { - result = append(result, tables[i]) - } - return result, err -} diff --git a/pkg/clickhouse/structs.go b/pkg/clickhouse/structs.go index dcbc7904..90e6bf48 100644 --- a/pkg/clickhouse/structs.go +++ b/pkg/clickhouse/structs.go @@ -19,7 +19,7 @@ type Table struct { Name string `ch:"name"` Engine string `ch:"engine"` // fields depends on `clickhouse-server` version - DataPath string `ch:"data_path"` // For legacy support + DataPath string `ch:"data_path"` DataPaths []string `ch:"data_paths"` UUID string `ch:"uuid"` CreateTableQuery string `ch:"create_table_query"` @@ -38,10 +38,12 @@ type IsSystemTablesFieldPresent struct { } type Disk struct { - Name string `ch:"name"` - Path string `ch:"path"` - Type string `ch:"type"` - IsBackup bool + Name string `ch:"name"` + Path string `ch:"path"` + Type string `ch:"type"` + FreeSpace uint64 `ch:"free_space"` + StoragePolicies []string `ch:"storage_policies"` + IsBackup bool } // Database - Clickhouse system.databases struct @@ -79,3 +81,23 @@ type SystemBackups struct { UncompressedSize uint64 `ch:"uncompressed_size"` NumFiles uint64 `ch:"num_files"` } + +// ColumnDataTypes - info from system.parts_columns +type ColumnDataTypes struct { + Column string `ch:"column"` + Types []string `ch:"uniq_types"` +} + +// BackupDataSize - info from system.parts or system.tables when embedded BACKUP statement return zero size +type BackupDataSize struct { + Size uint64 `ch:"backup_data_size"` +} + +type UserDirectory struct { + Name string `ch:"name"` +} + +type RBACObject struct { + Id string `ch:"id"` + Name string `ch:"name"` +} diff --git a/pkg/common/utils.go b/pkg/common/utils.go index 18de0b48..f126e0d4 100644 --- a/pkg/common/utils.go +++ b/pkg/common/utils.go @@ -6,8 +6,12 @@ import ( ) func TablePathEncode(str string) string { - return strings.NewReplacer(".", "%2E", "-", "%2D").Replace(url.PathEscape(str)) - + return strings.NewReplacer( + "!", "%21", "@", "%40", "#", "%23", "$", "%24", "^", "%5E", "&", "%26", "*", "%2A", + "(", "%28", ")", "%29", "+", "%2B", "-", "%2D", "=", "%3D", "[", "%5B", "]", "%5D", + "{", "%7B", "}", "%7D", "|", "%7C", ";", "%3B", "'", "%27", ":", "%3A", "\"", "%22", + ",", "%2C", ".", "%2E", "/", "%2F", "<", "%3C", ">", "%3E", "?", "%3F", "~", "%7E", + ).Replace(url.PathEscape(str)) } func SumMapValuesInt(m map[string]int) int { diff --git a/pkg/common/utils_test.go b/pkg/common/utils_test.go new file mode 100644 index 00000000..0f9336f0 --- /dev/null +++ b/pkg/common/utils_test.go @@ -0,0 +1,19 @@ +package common + +import ( + "github.com/stretchr/testify/require" + "net/url" + "testing" +) + +func TestTablePathEncode(t *testing.T) { + r := require.New(t) + str := `!@#$^&*()+-=[]{}|;':\",./<>?~` + expected := "%21%40%23%24%5E%26%2A%28%29%2B%2D%3D%5B%5D%7B%7D%7C%3B%27%3A%5C%22%2C%2E%2F%3C%3E%3F%7E" + + actual := TablePathEncode(str) + r.Equal(expected, actual) + decoded, err := url.PathUnescape(actual) + r.NoError(err) + r.Equal(str, decoded) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 2a651b15..b4556c18 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -5,11 +5,12 @@ import ( "fmt" "math" "os" + "regexp" "runtime" "strings" "time" - "github.com/Altinity/clickhouse-backup/pkg/log_helper" + "github.com/Altinity/clickhouse-backup/v2/pkg/log_helper" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/kelseyhightower/envconfig" "github.com/rs/zerolog/log" @@ -37,29 +38,36 @@ type Config struct { // GeneralConfig - general setting section type GeneralConfig struct { - RemoteStorage string `yaml:"remote_storage" envconfig:"REMOTE_STORAGE"` - MaxFileSize int64 `yaml:"max_file_size" envconfig:"MAX_FILE_SIZE"` - DisableProgressBar bool `yaml:"disable_progress_bar" envconfig:"DISABLE_PROGRESS_BAR"` - BackupsToKeepLocal int `yaml:"backups_to_keep_local" envconfig:"BACKUPS_TO_KEEP_LOCAL"` - BackupsToKeepRemote int `yaml:"backups_to_keep_remote" envconfig:"BACKUPS_TO_KEEP_REMOTE"` - LogLevel string `yaml:"log_level" envconfig:"LOG_LEVEL"` - AllowEmptyBackups bool `yaml:"allow_empty_backups" envconfig:"ALLOW_EMPTY_BACKUPS"` - DownloadConcurrency uint8 `yaml:"download_concurrency" envconfig:"DOWNLOAD_CONCURRENCY"` - UploadConcurrency uint8 `yaml:"upload_concurrency" envconfig:"UPLOAD_CONCURRENCY"` - UseResumableState bool `yaml:"use_resumable_state" envconfig:"USE_RESUMABLE_STATE"` - RestoreSchemaOnCluster string `yaml:"restore_schema_on_cluster" envconfig:"RESTORE_SCHEMA_ON_CLUSTER"` - UploadByPart bool `yaml:"upload_by_part" envconfig:"UPLOAD_BY_PART"` - DownloadByPart bool `yaml:"download_by_part" envconfig:"DOWNLOAD_BY_PART"` - RestoreDatabaseMapping map[string]string `yaml:"restore_database_mapping" envconfig:"RESTORE_DATABASE_MAPPING"` - RetriesOnFailure int `yaml:"retries_on_failure" envconfig:"RETRIES_ON_FAILURE"` - RetriesPause string `yaml:"retries_pause" envconfig:"RETRIES_PAUSE"` - WatchInterval string `yaml:"watch_interval" envconfig:"WATCH_INTERVAL"` - FullInterval string `yaml:"full_interval" envconfig:"FULL_INTERVAL"` - WatchBackupNameTemplate string `yaml:"watch_backup_name_template" envconfig:"WATCH_BACKUP_NAME_TEMPLATE"` - ShardedOperationMode string `yaml:"sharded_operation_mode" envconfig:"SHARDED_OPERATION_MODE"` - RetriesDuration time.Duration - WatchDuration time.Duration - FullDuration time.Duration + RemoteStorage string `yaml:"remote_storage" envconfig:"REMOTE_STORAGE"` + MaxFileSize int64 `yaml:"max_file_size" envconfig:"MAX_FILE_SIZE"` + BackupsToKeepLocal int `yaml:"backups_to_keep_local" envconfig:"BACKUPS_TO_KEEP_LOCAL"` + BackupsToKeepRemote int `yaml:"backups_to_keep_remote" envconfig:"BACKUPS_TO_KEEP_REMOTE"` + LogLevel string `yaml:"log_level" envconfig:"LOG_LEVEL"` + AllowEmptyBackups bool `yaml:"allow_empty_backups" envconfig:"ALLOW_EMPTY_BACKUPS"` + DownloadConcurrency uint8 `yaml:"download_concurrency" envconfig:"DOWNLOAD_CONCURRENCY"` + UploadConcurrency uint8 `yaml:"upload_concurrency" envconfig:"UPLOAD_CONCURRENCY"` + UploadMaxBytesPerSecond uint64 `yaml:"upload_max_bytes_per_second" envconfig:"UPLOAD_MAX_BYTES_PER_SECOND"` + DownloadMaxBytesPerSecond uint64 `yaml:"download_max_bytes_per_second" envconfig:"DOWNLOAD_MAX_BYTES_PER_SECOND"` + ObjectDiskServerSideCopyConcurrency uint8 `yaml:"object_disk_server_side_copy_concurrency" envconfig:"OBJECT_DISK_SERVER_SIDE_COPY_CONCURRENCY"` + UseResumableState bool `yaml:"use_resumable_state" envconfig:"USE_RESUMABLE_STATE"` + RestoreSchemaOnCluster string `yaml:"restore_schema_on_cluster" envconfig:"RESTORE_SCHEMA_ON_CLUSTER"` + UploadByPart bool `yaml:"upload_by_part" envconfig:"UPLOAD_BY_PART"` + DownloadByPart bool `yaml:"download_by_part" envconfig:"DOWNLOAD_BY_PART"` + RestoreDatabaseMapping map[string]string `yaml:"restore_database_mapping" envconfig:"RESTORE_DATABASE_MAPPING"` + RestoreTableMapping map[string]string `yaml:"restore_table_mapping" envconfig:"RESTORE_TABLE_MAPPING"` + RetriesOnFailure int `yaml:"retries_on_failure" envconfig:"RETRIES_ON_FAILURE"` + RetriesPause string `yaml:"retries_pause" envconfig:"RETRIES_PAUSE"` + WatchInterval string `yaml:"watch_interval" envconfig:"WATCH_INTERVAL"` + FullInterval string `yaml:"full_interval" envconfig:"FULL_INTERVAL"` + WatchBackupNameTemplate string `yaml:"watch_backup_name_template" envconfig:"WATCH_BACKUP_NAME_TEMPLATE"` + ShardedOperationMode string `yaml:"sharded_operation_mode" envconfig:"SHARDED_OPERATION_MODE"` + CPUNicePriority int `yaml:"cpu_nice_priority" envconfig:"CPU_NICE_PRIORITY"` + IONicePriority string `yaml:"io_nice_priority" envconfig:"IO_NICE_PRIORITY"` + RBACBackupAlways bool `yaml:"rbac_backup_always" envconfig:"RBAC_BACKUP_ALWAYS"` + RBACConflictResolution string `yaml:"rbac_conflict_resolution" envconfig:"RBAC_CONFLICT_RESOLUTION"` + RetriesDuration time.Duration + WatchDuration time.Duration + FullDuration time.Duration } // GCSConfig - GCS settings section @@ -67,16 +75,24 @@ type GCSConfig struct { CredentialsFile string `yaml:"credentials_file" envconfig:"GCS_CREDENTIALS_FILE"` CredentialsJSON string `yaml:"credentials_json" envconfig:"GCS_CREDENTIALS_JSON"` CredentialsJSONEncoded string `yaml:"credentials_json_encoded" envconfig:"GCS_CREDENTIALS_JSON_ENCODED"` + EmbeddedAccessKey string `yaml:"embedded_access_key" envconfig:"GCS_EMBEDDED_ACCESS_KEY"` + EmbeddedSecretKey string `yaml:"embedded_secret_key" envconfig:"GCS_EMBEDDED_SECRET_KEY"` + SkipCredentials bool `yaml:"skip_credentials" envconfig:"GCS_SKIP_CREDENTIALS"` Bucket string `yaml:"bucket" envconfig:"GCS_BUCKET"` Path string `yaml:"path" envconfig:"GCS_PATH"` ObjectDiskPath string `yaml:"object_disk_path" envconfig:"GCS_OBJECT_DISK_PATH"` CompressionLevel int `yaml:"compression_level" envconfig:"GCS_COMPRESSION_LEVEL"` CompressionFormat string `yaml:"compression_format" envconfig:"GCS_COMPRESSION_FORMAT"` Debug bool `yaml:"debug" envconfig:"GCS_DEBUG"` + ForceHttp bool `yaml:"force_http" envconfig:"GCS_FORCE_HTTP"` Endpoint string `yaml:"endpoint" envconfig:"GCS_ENDPOINT"` StorageClass string `yaml:"storage_class" envconfig:"GCS_STORAGE_CLASS"` ObjectLabels map[string]string `yaml:"object_labels" envconfig:"GCS_OBJECT_LABELS"` CustomStorageClassMap map[string]string `yaml:"custom_storage_class_map" envconfig:"GCS_CUSTOM_STORAGE_CLASS_MAP"` + // NOTE: ClientPoolSize should be at least 2 times bigger than + // UploadConcurrency or DownloadConcurrency in each upload and download case + ClientPoolSize int `yaml:"client_pool_size" envconfig:"GCS_CLIENT_POOL_SIZE"` + ChunkSize int `yaml:"chunk_size" envconfig:"GCS_CHUNK_SIZE"` } // AzureBlobConfig - Azure Blob settings section @@ -97,6 +113,7 @@ type AzureBlobConfig struct { MaxBuffers int `yaml:"buffer_count" envconfig:"AZBLOB_MAX_BUFFERS"` MaxPartsCount int `yaml:"max_parts_count" envconfig:"AZBLOB_MAX_PARTS_COUNT"` Timeout string `yaml:"timeout" envconfig:"AZBLOB_TIMEOUT"` + Debug bool `yaml:"debug" envconfig:"AZBLOB_DEBUG"` } // S3Config - s3 settings section @@ -129,6 +146,8 @@ type S3Config struct { MaxPartsCount int64 `yaml:"max_parts_count" envconfig:"S3_MAX_PARTS_COUNT"` AllowMultipartDownload bool `yaml:"allow_multipart_download" envconfig:"S3_ALLOW_MULTIPART_DOWNLOAD"` ObjectLabels map[string]string `yaml:"object_labels" envconfig:"S3_OBJECT_LABELS"` + RequestPayer string `yaml:"request_payer" envconfig:"S3_REQUEST_PAYER"` + CheckSumAlgorithm string `yaml:"check_sum_algorithm" envconfig:"S3_CHECKSUM_ALGORITHM"` Debug bool `yaml:"debug" envconfig:"S3_DEBUG"` } @@ -151,6 +170,7 @@ type FTPConfig struct { Username string `yaml:"username" envconfig:"FTP_USERNAME"` Password string `yaml:"password" envconfig:"FTP_PASSWORD"` TLS bool `yaml:"tls" envconfig:"FTP_TLS"` + SkipTLSVerify bool `yaml:"skip_tls_verify" envconfig:"FTP_SKIP_TLS_VERIFY"` Path string `yaml:"path" envconfig:"FTP_PATH"` ObjectDiskPath string `yaml:"object_disk_path" envconfig:"FTP_OBJECT_DISK_PATH"` CompressionFormat string `yaml:"compression_format" envconfig:"FTP_COMPRESSION_FORMAT"` @@ -212,6 +232,7 @@ type ClickHouseConfig struct { TLSKey string `yaml:"tls_key" envconfig:"CLICKHOUSE_TLS_KEY"` TLSCert string `yaml:"tls_cert" envconfig:"CLICKHOUSE_TLS_CERT"` TLSCa string `yaml:"tls_ca" envconfig:"CLICKHOUSE_TLS_CA"` + MaxConnections int `yaml:"max_connections" envconfig:"CLICKHOUSE_MAX_CONNECTIONS"` Debug bool `yaml:"debug" envconfig:"CLICKHOUSE_DEBUG"` } @@ -230,6 +251,7 @@ type APIConfig struct { IntegrationTablesHost string `yaml:"integration_tables_host" envconfig:"API_INTEGRATION_TABLES_HOST"` AllowParallel bool `yaml:"allow_parallel" envconfig:"API_ALLOW_PARALLEL"` CompleteResumableAfterRestart bool `yaml:"complete_resumable_after_restart" envconfig:"API_COMPLETE_RESUMABLE_AFTER_RESTART"` + WatchIsMainProcess bool `yaml:"watch_is_main_process" envconfig:"WATCH_IS_MAIN_PROCESS"` } // ArchiveExtensions - list of available compression formats and associated file extensions @@ -285,6 +307,8 @@ func (cfg *Config) GetCompressionFormat() string { } } +var freezeByPartBeginAndRE = regexp.MustCompile(`(?im)^\s*AND\s+`) + // LoadConfig - load config from file + environment variables func LoadConfig(configLocation string) (*Config, error) { cfg := DefaultConfig() @@ -299,7 +323,7 @@ func LoadConfig(configLocation string) (*Config, error) { return nil, err } - //adjust upload_concurrency for storage types which not have SDK level concurrency, https://github.com/Altinity/clickhouse-backup/issues/658 + //auto-tuning upload_concurrency for storage types which not have SDK level concurrency, https://github.com/Altinity/clickhouse-backup/issues/658 cfgWithoutDefault := &Config{} if err := yaml.Unmarshal(configYaml, &cfgWithoutDefault); err != nil { return nil, fmt.Errorf("can't parse config file: %v", err) @@ -310,11 +334,29 @@ func LoadConfig(configLocation string) (*Config, error) { if (cfg.General.RemoteStorage == "gcs" || cfg.General.RemoteStorage == "azblob" || cfg.General.RemoteStorage == "cos") && cfgWithoutDefault.General.UploadConcurrency == 0 { cfg.General.UploadConcurrency = uint8(runtime.NumCPU() / 2) } - cfg.AzureBlob.Path = strings.TrimPrefix(cfg.AzureBlob.Path, "/") - cfg.S3.Path = strings.TrimPrefix(cfg.S3.Path, "/") - cfg.GCS.Path = strings.TrimPrefix(cfg.GCS.Path, "/") + cfg.AzureBlob.Path = strings.Trim(cfg.AzureBlob.Path, "/ ") + cfg.S3.Path = strings.Trim(cfg.S3.Path, "/ ") + cfg.GCS.Path = strings.Trim(cfg.GCS.Path, "/ ") + + cfg.S3.ObjectDiskPath = strings.Trim(cfg.S3.ObjectDiskPath,"/ ") + cfg.GCS.ObjectDiskPath = strings.Trim(cfg.GCS.ObjectDiskPath,"/ ") + cfg.AzureBlob.ObjectDiskPath = strings.Trim(cfg.AzureBlob.ObjectDiskPath,"/ ") + + // https://github.com/Altinity/clickhouse-backup/issues/855 + if cfg.ClickHouse.FreezeByPart && cfg.ClickHouse.FreezeByPartWhere != "" && !freezeByPartBeginAndRE.MatchString(cfg.ClickHouse.FreezeByPartWhere) { + cfg.ClickHouse.FreezeByPartWhere = " AND " + cfg.ClickHouse.FreezeByPartWhere + } + + log_helper.SetLogLevelFromString(cfg.General.LogLevel) - return cfg, ValidateConfig(cfg) + + if err = ValidateConfig(cfg); err != nil { + return cfg, err + } + if err = cfg.SetPriority(); err != nil { + return cfg, err + } + return cfg, nil } func ValidateConfig(cfg *Config) error { @@ -472,27 +514,32 @@ func DefaultConfig() *Config { } return &Config{ General: GeneralConfig{ - RemoteStorage: "none", - MaxFileSize: 0, - BackupsToKeepLocal: 0, - BackupsToKeepRemote: 0, - LogLevel: "info", - DisableProgressBar: true, - UploadConcurrency: uploadConcurrency, - DownloadConcurrency: downloadConcurrency, - RestoreSchemaOnCluster: "", - UploadByPart: true, - DownloadByPart: true, - UseResumableState: true, - RetriesOnFailure: 3, - RetriesPause: "30s", - RetriesDuration: 100 * time.Millisecond, - WatchInterval: "1h", - WatchDuration: 1 * time.Hour, - FullInterval: "24h", - FullDuration: 24 * time.Hour, - WatchBackupNameTemplate: "shard{shard}-{type}-{time:20060102150405}", - RestoreDatabaseMapping: make(map[string]string, 0), + RemoteStorage: "none", + MaxFileSize: 0, + BackupsToKeepLocal: 0, + BackupsToKeepRemote: 0, + LogLevel: "info", + UploadConcurrency: uploadConcurrency, + DownloadConcurrency: downloadConcurrency, + ObjectDiskServerSideCopyConcurrency: 32, + RestoreSchemaOnCluster: "", + UploadByPart: true, + DownloadByPart: true, + UseResumableState: true, + RetriesOnFailure: 3, + RetriesPause: "30s", + RetriesDuration: 100 * time.Millisecond, + WatchInterval: "1h", + WatchDuration: 1 * time.Hour, + FullInterval: "24h", + FullDuration: 24 * time.Hour, + WatchBackupNameTemplate: "shard{shard}-{type}-{time:20060102150405}", + RestoreDatabaseMapping: make(map[string]string), + RestoreTableMapping: make(map[string]string), + IONicePriority: "idle", + CPUNicePriority: 15, + RBACBackupAlways: true, + RBACConflictResolution: "recreate", }, ClickHouse: ClickHouseConfig{ Username: "default", @@ -505,7 +552,7 @@ func DefaultConfig() *Config { "information_schema.*", "_temporary_and_external_tables.*", }, - Timeout: "5m", + Timeout: "30m", SyncReplicatedTables: false, LogSQLQueries: true, ConfigDir: "/etc/clickhouse-server/", @@ -516,6 +563,7 @@ func DefaultConfig() *Config { BackupMutations: true, RestoreAsAttach: false, CheckPartsColumns: true, + MaxConnections: int(downloadConcurrency), }, AzureBlob: AzureBlobConfig{ EndpointSchema: "https", @@ -524,8 +572,8 @@ func DefaultConfig() *Config { CompressionFormat: "tar", BufferSize: 0, MaxBuffers: 3, - MaxPartsCount: 5000, - Timeout: "15m", + MaxPartsCount: 256, + Timeout: "4h", }, S3: S3Config{ Region: "us-east-1", @@ -539,12 +587,13 @@ func DefaultConfig() *Config { StorageClass: string(s3types.StorageClassStandard), Concurrency: int(downloadConcurrency + 1), PartSize: 0, - MaxPartsCount: 5000, + MaxPartsCount: 4000, }, GCS: GCSConfig{ CompressionLevel: 1, CompressionFormat: "tar", StorageClass: "STANDARD", + ClientPoolSize: int(max(uploadConcurrency, downloadConcurrency)) * 3, }, COS: COSConfig{ RowURL: "", @@ -562,7 +611,7 @@ func DefaultConfig() *Config { }, FTP: FTPConfig{ Timeout: "2m", - Concurrency: downloadConcurrency + 1, + Concurrency: downloadConcurrency * 3, CompressionFormat: "tar", CompressionLevel: 1, }, @@ -570,7 +619,7 @@ func DefaultConfig() *Config { Port: 22, CompressionFormat: "tar", CompressionLevel: 1, - Concurrency: int(downloadConcurrency + 1), + Concurrency: int(downloadConcurrency * 3), }, Custom: CustomConfig{ CommandTimeout: "4h", @@ -580,11 +629,13 @@ func DefaultConfig() *Config { } func GetConfigFromCli(ctx *cli.Context) *Config { + oldEnvValues := OverrideEnvVars(ctx) configPath := GetConfigPath(ctx) cfg, err := LoadConfig(configPath) if err != nil { log.Fatal().Stack().Err(err).Send() } + RestoreEnvVars(oldEnvValues) return cfg } @@ -600,3 +651,45 @@ func GetConfigPath(ctx *cli.Context) string { } return DefaultConfigPath } + +type oldEnvValues struct { + OldValue string + WasPresent bool +} + +func OverrideEnvVars(ctx *cli.Context) map[string]oldEnvValues { + env := ctx.StringSlice("env") + oldValues := map[string]oldEnvValues{} + if len(env) > 0 { + for _, v := range env { + envVariable := strings.SplitN(v, "=", 2) + if len(envVariable) < 2 { + envVariable = append(envVariable, "true") + } + log.Info().Msgf("override %s=%s", envVariable[0], envVariable[1]) + oldValue, wasPresent := os.LookupEnv(envVariable[0]) + oldValues[envVariable[0]] = oldEnvValues{ + OldValue: oldValue, + WasPresent: wasPresent, + } + if err := os.Setenv(envVariable[0], envVariable[1]); err != nil { + log.Warn().Msgf("can't override %s=%s, error: %v", envVariable[0], envVariable[1], err) + } + } + } + return oldValues +} + +func RestoreEnvVars(envVars map[string]oldEnvValues) { + for name, oldEnv := range envVars { + if oldEnv.WasPresent { + if err := os.Setenv(name, oldEnv.OldValue); err != nil { + log.Warn().Msgf("RestoreEnvVars can't restore %s=%s, error: %v", name, oldEnv.OldValue, err) + } + } else { + if err := os.Unsetenv(name); err != nil { + log.Warn().Msgf("RestoreEnvVars can't delete %s, error: %v", name, err) + } + } + } +} diff --git a/pkg/config/config_darwin.go b/pkg/config/config_darwin.go new file mode 100644 index 00000000..a886e886 --- /dev/null +++ b/pkg/config/config_darwin.go @@ -0,0 +1,19 @@ +package config + +import ( + "github.com/apex/log" + "syscall" + "os" +) + +func (cfg *Config) SetPriority() error { + var executable string + if err := syscall.Setpriority(0, 0, cfg.General.CPUNicePriority); err != nil { + executable, err = os.Executable() + if err != nil { + log.Warnf("can't get current executable path: %v", err) + } + log.Warnf("can't set CPU priority %s, error: %v, use `sudo setcap cap_sys_nice+ep %s` to fix it", cfg.General.CPUNicePriority, err, executable) + } + return nil +} diff --git a/pkg/config/config_linux.go b/pkg/config/config_linux.go new file mode 100644 index 00000000..1192da9a --- /dev/null +++ b/pkg/config/config_linux.go @@ -0,0 +1,29 @@ +package config + +import ( + "github.com/apex/log" + "github.com/xyproto/gionice" + "os" +) + +func (cfg *Config) SetPriority() error { + var err error + var executable string + if cfg.General.IONicePriority != "" { + var nicePriority gionice.PriClass + executable, err = os.Executable() + if err != nil { + log.Warnf("can't get current executable path: %v", err) + } + if nicePriority, err = gionice.Parse(cfg.General.IONicePriority); err != nil { + return err + } + if err = gionice.SetIDPri(0, nicePriority, 7, gionice.IOPRIO_WHO_PGRP); err != nil { + log.Warnf("can't set i/o priority %s, error: %v, use `sudo setcap cap_sys_nice+ep %s` to fix it", cfg.General.IONicePriority, err, executable) + } + } + if err = gionice.SetNicePri(0, gionice.PRIO_PROCESS, cfg.General.CPUNicePriority); err != nil { + log.Warnf("can't set CPU priority %v, error: %v, use `sudo setcap cap_sys_nice+ep %s` to fix it", cfg.General.CPUNicePriority, err, executable) + } + return nil +} diff --git a/pkg/custom/delete_custom.go b/pkg/custom/delete_custom.go index d737865f..7c06d26a 100644 --- a/pkg/custom/delete_custom.go +++ b/pkg/custom/delete_custom.go @@ -3,8 +3,8 @@ package custom import ( "context" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/utils" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" "github.com/rs/zerolog/log" "time" ) diff --git a/pkg/custom/download_custom.go b/pkg/custom/download_custom.go index 64b16230..a774ec3b 100644 --- a/pkg/custom/download_custom.go +++ b/pkg/custom/download_custom.go @@ -3,8 +3,8 @@ package custom import ( "context" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/utils" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" "github.com/eapache/go-resiliency/retrier" "github.com/rs/zerolog/log" "time" diff --git a/pkg/custom/list_custom.go b/pkg/custom/list_custom.go index 57e28735..b740add6 100644 --- a/pkg/custom/list_custom.go +++ b/pkg/custom/list_custom.go @@ -4,9 +4,9 @@ import ( "context" "encoding/json" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/storage" - "github.com/Altinity/clickhouse-backup/pkg/utils" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" "github.com/rs/zerolog/log" "strings" "time" @@ -23,7 +23,12 @@ func List(ctx context.Context, cfg *config.Config) ([]storage.Backup, error) { args := ApplyCommandTemplate(cfg.Custom.ListCommand, templateData) out, err := utils.ExecCmdOut(ctx, cfg.Custom.CommandTimeoutDuration, args[0], args[1:]...) if err == nil { - outLines := strings.Split(strings.TrimRight(out, "\n"), "\n") + outLines := make([]string, 0) + outTrim := strings.TrimRight(out, "\n") + if outTrim != "" { + outLines = strings.Split(outTrim, "\n") + } + backupList := make([]storage.Backup, len(outLines)) for i, line := range outLines { if len(line) > 0 { diff --git a/pkg/custom/upload_custom.go b/pkg/custom/upload_custom.go index 24bff2a9..f3383b81 100644 --- a/pkg/custom/upload_custom.go +++ b/pkg/custom/upload_custom.go @@ -3,8 +3,8 @@ package custom import ( "context" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/utils" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" "github.com/eapache/go-resiliency/retrier" "github.com/rs/zerolog/log" "time" diff --git a/pkg/filesystemhelper/filesystemhelper.go b/pkg/filesystemhelper/filesystemhelper.go index bc4e5f16..26a7245e 100644 --- a/pkg/filesystemhelper/filesystemhelper.go +++ b/pkg/filesystemhelper/filesystemhelper.go @@ -2,6 +2,7 @@ package filesystemhelper import ( "fmt" + "net/url" "os" "path" "path/filepath" @@ -10,10 +11,10 @@ import ( "syscall" "time" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/common" - "github.com/Altinity/clickhouse-backup/pkg/metadata" - "github.com/Altinity/clickhouse-backup/pkg/utils" + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/common" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" "github.com/rs/zerolog/log" ) @@ -113,25 +114,46 @@ func MkdirAll(path string, ch *clickhouse.ClickHouse, disks []clickhouse.Disk) e return nil } -// HardlinkBackupPartsToStorage - copy parts for specific table to detached folder -func HardlinkBackupPartsToStorage(backupName string, backupTable metadata.TableMetadata, disks []clickhouse.Disk, tableDataPaths []string, ch *clickhouse.ClickHouse, toDetached bool) error { +// HardlinkBackupPartsToStorage - copy partitions for specific table to detached folder +func HardlinkBackupPartsToStorage(backupName string, backupTable metadata.TableMetadata, disks []clickhouse.Disk, diskMap map[string]string, tableDataPaths []string, ch *clickhouse.ClickHouse, toDetached bool) error { start := time.Now() dstDataPaths := clickhouse.GetDisksByPaths(disks, tableDataPaths) dbAndTableDir := path.Join(common.TablePathEncode(backupTable.Database), common.TablePathEncode(backupTable.Table)) - for _, backupDisk := range disks { - backupDiskName := backupDisk.Name - if len(backupTable.Parts[backupDiskName]) == 0 { - log.Debug().Msgf("%s disk have no parts", backupDisk.Name) - continue - } - dstParentDir, dstParentDirExists := dstDataPaths[backupDiskName] - if !dstParentDirExists { - return fmt.Errorf("dstDataPaths=%#v, not contains %s", dstDataPaths, backupDiskName) - } - if toDetached { - dstParentDir = filepath.Join(dstParentDir, "detached") + if !toDetached { + for backupDiskName := range backupTable.Parts { + dstParentDir, dstParentDirExists := dstDataPaths[backupDiskName] + if dstParentDirExists { + // avoid to restore to non-empty to avoid attach in already dropped partitions, corner case + existsFiles, err := os.ReadDir(dstParentDir) + if err != nil && !os.IsNotExist(err) { + return err + } + for _, f := range existsFiles { + if f.Name() != "detached" && !strings.HasSuffix(f.Name(), ".txt") { + return fmt.Errorf("%s contains exists data %v, we can't restore directly via ATTACH TABLE, use `clickhouse->restore_as_attach=false` in your config", dstParentDir, existsFiles) + } + } + } } + } + for backupDiskName := range backupTable.Parts { for _, part := range backupTable.Parts[backupDiskName] { + dstParentDir, dstParentDirExists := dstDataPaths[backupDiskName] + if !dstParentDirExists && part.RebalancedDisk == "" { + return fmt.Errorf("dstDataPaths=%#v, not contains %s", dstDataPaths, backupDiskName) + } + if !dstParentDirExists && part.RebalancedDisk != "" { + backupDiskName = part.RebalancedDisk + dstParentDir, dstParentDirExists = dstDataPaths[part.RebalancedDisk] + if !dstParentDirExists { + return fmt.Errorf("dstDataPaths=%#v, not contains %s", dstDataPaths, part.RebalancedDisk) + } + } + backupDiskPath := diskMap[backupDiskName] + if toDetached { + dstParentDir = filepath.Join(dstParentDir, "detached") + + } dstPartPath := filepath.Join(dstParentDir, part.Name) info, err := os.Stat(dstPartPath) if err != nil { @@ -146,16 +168,16 @@ func HardlinkBackupPartsToStorage(backupName string, backupTable metadata.TableM } else if !info.IsDir() { return fmt.Errorf("'%s' should be directory or absent", dstPartPath) } - partPath := path.Join(backupDisk.Path, "backup", backupName, "shadow", dbAndTableDir, backupDisk.Name, part.Name) - // Legacy backup support - if _, err := os.Stat(partPath); os.IsNotExist(err) { - partPath = path.Join(backupDisk.Path, "backup", backupName, "shadow", dbAndTableDir, part.Name) - } - if err := filepath.Walk(partPath, func(filePath string, info os.FileInfo, err error) error { + srcPartPath := path.Join(backupDiskPath, "backup", backupName, "shadow", dbAndTableDir, backupDiskName, part.Name) + if err := filepath.Walk(srcPartPath, func(filePath string, info os.FileInfo, err error) error { if err != nil { return err } - filename := strings.Trim(strings.TrimPrefix(filePath, partPath), "/") + // fix https://github.com/Altinity/clickhouse-backup/issues/826 + if strings.Contains(info.Name(), "frozen_metadata") { + return nil + } + filename := strings.Trim(strings.TrimPrefix(filePath, srcPartPath), "/") dstFilePath := filepath.Join(dstPartPath, filename) if info.IsDir() { log.Debug().Msgf("MkDir %s", dstFilePath) @@ -182,20 +204,58 @@ func HardlinkBackupPartsToStorage(backupName string, backupTable metadata.TableM } func IsPartInPartition(partName string, partitionsBackupMap common.EmptyMap) bool { - _, ok := partitionsBackupMap[strings.Split(partName, "_")[0]] - return ok + partitionId := strings.Split(partName, "_")[0] + if _, exists := partitionsBackupMap[partitionId]; exists { + return true + } + for pattern := range partitionsBackupMap { + if matched, err := filepath.Match(pattern, partitionId); err == nil && matched { + return true + } else if err != nil { + log.Warn().Msgf("error filepath.Match(%s, %s) error: %v", pattern, partitionId, err) + log.Debug().Msgf("%s not found in %s, file will filtered", partitionId, partitionsBackupMap) + return false + } + } + return false } func IsFileInPartition(disk, fileName string, partitionsBackupMap common.EmptyMap) bool { fileName = strings.TrimPrefix(fileName, disk+"_") - _, ok := partitionsBackupMap[strings.Split(fileName, "_")[0]] - return ok + fileName = strings.Split(fileName, "_")[0] + if strings.Contains(fileName, "%") { + decodedFileName, err := url.QueryUnescape(fileName) + if err != nil { + log.Warn().Msgf("error decoding %s: %v", fileName, err) + log.Debug().Msgf("%s not found in %s, file will filtered", fileName, partitionsBackupMap) + return false + } + fileName = decodedFileName + } + if _, exists := partitionsBackupMap[fileName]; exists { + return true + } + for pattern := range partitionsBackupMap { + if matched, err := filepath.Match(pattern, fileName); err == nil && matched { + return true + } else if err != nil { + log.Warn().Msgf("error filepath.Match(%s, %s) error: %v", pattern, fileName, err) + log.Debug().Msgf("%s not found in %s, file will filtered", fileName, partitionsBackupMap) + return false + } + } + return false } -func MoveShadow(shadowPath, backupPartsPath string, partitionsBackupMap common.EmptyMap) ([]metadata.Part, int64, error) { +func MoveShadowToBackup(shadowPath, backupPartsPath string, partitionsBackupMap common.EmptyMap, tableDiffFromRemote metadata.TableMetadata, disk clickhouse.Disk, version int) ([]metadata.Part, int64, error) { size := int64(0) parts := make([]metadata.Part, 0) err := filepath.Walk(shadowPath, func(filePath string, info os.FileInfo, err error) error { + // fix https://github.com/Altinity/clickhouse-backup/issues/826 + if strings.Contains(info.Name(), "frozen_metadata") { + return nil + } + // possible relative path // store / 1f9 / 1f9dc899-0de9-41f8-b95c-26c1f0d67d93 / 20181023_2_2_0 / checksums.txt // store / 1f9 / 1f9dc899-0de9-41f8-b95c-26c1f0d67d93 / 20181023_2_2_0 / x.proj / checksums.txt @@ -209,9 +269,16 @@ func MoveShadow(shadowPath, backupPartsPath string, partitionsBackupMap common.E if len(partitionsBackupMap) != 0 && !IsPartInPartition(pathParts[3], partitionsBackupMap) { return nil } + var isRequiredPartFound, partExists bool + if tableDiffFromRemote.Database != "" && tableDiffFromRemote.Table != "" && len(tableDiffFromRemote.Parts) > 0 && len(tableDiffFromRemote.Parts[disk.Name]) > 0 { + parts, isRequiredPartFound, partExists = addRequiredPartIfNotExists(parts, pathParts[3], tableDiffFromRemote, disk) + if isRequiredPartFound { + return nil + } + } dstFilePath := filepath.Join(backupPartsPath, pathParts[3]) if info.IsDir() { - if !strings.HasSuffix(pathParts[3], ".proj") { + if !strings.HasSuffix(pathParts[3], ".proj") && !isRequiredPartFound && !partExists { parts = append(parts, metadata.Part{ Name: pathParts[3], }) @@ -223,11 +290,38 @@ func MoveShadow(shadowPath, backupPartsPath string, partitionsBackupMap common.E return nil } size += info.Size() - return os.Rename(filePath, dstFilePath) + if version < 21004000 { + return os.Rename(filePath, dstFilePath) + } else { + return os.Link(filePath, dstFilePath) + } }) return parts, size, err } +func addRequiredPartIfNotExists(parts []metadata.Part, relativePath string, tableDiffFromRemote metadata.TableMetadata, disk clickhouse.Disk) ([]metadata.Part, bool, bool) { + isRequiredPartFound := false + exists := false + for _, diffPart := range tableDiffFromRemote.Parts[disk.Name] { + if diffPart.Name == relativePath || strings.HasPrefix(relativePath, diffPart.Name+"/") { + for _, p := range parts { + if p.Name == relativePath || strings.HasPrefix(relativePath, p.Name+"/") { + exists = true + break + } + } + if !exists { + parts = append(parts, metadata.Part{ + Name: relativePath, + Required: true, + }) + } + isRequiredPartFound = true + } + } + return parts, isRequiredPartFound, exists +} + func IsDuplicatedParts(part1, part2 string) error { p1, err := os.Open(part1) if err != nil { diff --git a/pkg/keeper/keeper.go b/pkg/keeper/keeper.go index d75d9a00..20cc42af 100644 --- a/pkg/keeper/keeper.go +++ b/pkg/keeper/keeper.go @@ -14,7 +14,7 @@ import ( "strings" "time" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" "github.com/go-zookeeper/zk" ) @@ -37,14 +37,13 @@ func (KeeperLogToApexLogAdapter LogKeeperToApexLogAdapter) Printf(msg string, ar } } -type keeperDumpNode struct { +type DumpNode struct { Path string `json:"path"` Value string `json:"value"` } type Keeper struct { conn *zk.Conn - Log zerolog.Logger root string doc *xmlquery.Node xmlConfigFile string @@ -67,7 +66,7 @@ func (k *Keeper) Connect(ctx context.Context, ch *clickhouse.ClickHouse) error { if sessionTimeoutMs, err := strconv.ParseInt(sessionTimeoutMsNode.InnerText(), 10, 64); err == nil { sessionTimeout = time.Duration(sessionTimeoutMs) * time.Millisecond } else { - k.Log.Warn().Msgf("can't parse /zookeeper/session_timeout_ms in %s, value: %v, error: %v ", configFile, sessionTimeoutMsNode.InnerText(), err) + log.Warn().Msgf("can't parse /zookeeper/session_timeout_ms in %s, value: %v, error: %v ", configFile, sessionTimeoutMsNode.InnerText(), err) } } nodeList := zookeeperNode.SelectElements("node") @@ -119,7 +118,7 @@ func (k *Keeper) Dump(prefix, dumpFile string) (int, error) { } defer func() { if err = f.Close(); err != nil { - k.Log.Warn().Msgf("can't close %s: %v", dumpFile, err) + log.Warn().Msgf("can't close %s: %v", dumpFile, err) } }() if !strings.HasPrefix(prefix, "/") && k.root != "" { @@ -132,20 +131,25 @@ func (k *Keeper) Dump(prefix, dumpFile string) (int, error) { return bytes, nil } +func (k *Keeper) ChildCount(prefix, nodePath string) (int, error) { + childrenNodes, _, err := k.conn.Children(path.Join(prefix, nodePath)) + return len(childrenNodes), err +} + func (k *Keeper) dumpNodeRecursive(prefix, nodePath string, f *os.File) (int, error) { value, _, err := k.conn.Get(path.Join(prefix, nodePath)) if err != nil { return 0, err } - bytes, err := k.writeJsonString(f, keeperDumpNode{Path: nodePath, Value: string(value)}) + bytes, err := k.writeJsonString(f, DumpNode{Path: nodePath, Value: string(value)}) if err != nil { return 0, err } - childs, _, err := k.conn.Children(path.Join(prefix, nodePath)) + children, _, err := k.conn.Children(path.Join(prefix, nodePath)) if err != nil { return 0, err } - for _, childPath := range childs { + for _, childPath := range children { if childBytes, err := k.dumpNodeRecursive(prefix, path.Join(nodePath, childPath), f); err != nil { return 0, err } else { @@ -155,7 +159,7 @@ func (k *Keeper) dumpNodeRecursive(prefix, nodePath string, f *os.File) (int, er return bytes, nil } -func (k *Keeper) writeJsonString(f *os.File, node keeperDumpNode) (int, error) { +func (k *Keeper) writeJsonString(f *os.File, node DumpNode) (int, error) { jsonLine, err := json.Marshal(node) if err != nil { return 0, err @@ -175,7 +179,7 @@ func (k *Keeper) Restore(dumpFile, prefix string) error { } defer func() { if err = f.Close(); err != nil { - k.Log.Warn().Msgf("can't close %s: %v", dumpFile, err) + log.Warn().Msgf("can't close %s: %v", dumpFile, err) } }() if !strings.HasPrefix(prefix, "/") && k.root != "" { @@ -183,7 +187,7 @@ func (k *Keeper) Restore(dumpFile, prefix string) error { } scanner := bufio.NewScanner(f) for scanner.Scan() { - node := keeperDumpNode{} + node := DumpNode{} if err = json.Unmarshal(scanner.Bytes(), &node); err != nil { return err } @@ -207,6 +211,39 @@ func (k *Keeper) Restore(dumpFile, prefix string) error { return nil } +type WalkCallBack = func(node DumpNode) (bool, error) + +func (k *Keeper) Walk(prefix, relativePath string, recursive bool, callback WalkCallBack) error { + nodePath := path.Join(prefix, relativePath) + value, stat, err := k.conn.Get(nodePath) + log.Debug().Msgf("Walk->get(%s) = %v, err = %v", nodePath, string(value), err) + if err != nil { + return err + } + var isDone bool + if isDone, err = callback(DumpNode{Path: nodePath, Value: string(value)}); err != nil { + return err + } + if isDone { + return nil + } + if recursive && stat.NumChildren > 0 { + children, _, err := k.conn.Children(path.Join(prefix, relativePath)) + if err != nil { + return err + } + for _, childPath := range children { + if childErr := k.Walk(prefix, path.Join(relativePath, childPath), recursive, callback); childErr != nil { + return childErr + } + } + } + return nil +} + +func (k *Keeper) Delete(nodePath string) error { + return k.conn.Delete(nodePath, -1) +} func (k *Keeper) Close() { k.conn.Close() } diff --git a/pkg/metadata/backup_metadata.go b/pkg/metadata/backup_metadata.go new file mode 100644 index 00000000..dd94903a --- /dev/null +++ b/pkg/metadata/backup_metadata.go @@ -0,0 +1,55 @@ +package metadata + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "time" +) + +type BackupMetadata struct { + BackupName string `json:"backup_name"` + Disks map[string]string `json:"disks"` // "default": "/var/lib/clickhouse" + DiskTypes map[string]string `json:"disk_types"` // "default": "local" + ClickhouseBackupVersion string `json:"version"` + CreationDate time.Time `json:"creation_date"` + Tags string `json:"tags,omitempty"` // "regular,embedded" + ClickHouseVersion string `json:"clickhouse_version,omitempty"` + DataSize uint64 `json:"data_size,omitempty"` + ObjectDiskSize uint64 `json:"object_disk_size,omitempty"` + MetadataSize uint64 `json:"metadata_size"` + RBACSize uint64 `json:"rbac_size,omitempty"` + ConfigSize uint64 `json:"config_size,omitempty"` + CompressedSize uint64 `json:"compressed_size,omitempty"` + Databases []DatabasesMeta `json:"databases,omitempty"` + Tables []TableTitle `json:"tables"` + Functions []FunctionsMeta `json:"functions"` + DataFormat string `json:"data_format"` + RequiredBackup string `json:"required_backup,omitempty"` +} + +func (b *BackupMetadata) GetFullSize() uint64 { + size := b.MetadataSize + b.ConfigSize + b.RBACSize + if strings.Contains(b.Tags, "embedded") { + size += b.DataSize + b.CompressedSize + } else { + if b.CompressedSize > 0 { + size += b.CompressedSize + b.ObjectDiskSize + } else { + size += b.DataSize + b.ObjectDiskSize + } + } + return size +} + +func (b *BackupMetadata) Save(location string) error { + tbBody, err := json.MarshalIndent(b, "", "\t") + if err != nil { + return fmt.Errorf("can't marshall backup metadata: %v", err) + } + if err := os.WriteFile(location, tbBody, 0640); err != nil { + return fmt.Errorf("can't save backup metadata: %v", err) + } + return nil +} diff --git a/pkg/metadata/load.go b/pkg/metadata/load.go deleted file mode 100644 index 2e18cf4d..00000000 --- a/pkg/metadata/load.go +++ /dev/null @@ -1,19 +0,0 @@ -package metadata - -import ( - "encoding/json" - "github.com/rs/zerolog/log" - "os" -) - -func (tm *TableMetadata) Load(location string) (uint64, error) { - data, err := os.ReadFile(location) - if err != nil { - return 0, err - } - if err := json.Unmarshal(data, tm); err != nil { - return 0, err - } - log.Debug().Str("operation", "TableMetadata.Load").Msgf("success %s", location) - return uint64(len(data)), nil -} diff --git a/pkg/metadata/metadata.go b/pkg/metadata/metadata.go index 98942884..fdbd154e 100644 --- a/pkg/metadata/metadata.go +++ b/pkg/metadata/metadata.go @@ -1,34 +1,10 @@ package metadata -import ( - "time" -) - type TableTitle struct { Database string `json:"database"` Table string `json:"table"` } -type BackupMetadata struct { - BackupName string `json:"backup_name"` - Disks map[string]string `json:"disks"` // "default": "/var/lib/clickhouse" - DiskTypes map[string]string `json:"disk_types"` // "default": "local" - ClickhouseBackupVersion string `json:"version"` - CreationDate time.Time `json:"creation_date"` - Tags string `json:"tags,omitempty"` // "regular,embedded" - ClickHouseVersion string `json:"clickhouse_version,omitempty"` - DataSize uint64 `json:"data_size,omitempty"` - MetadataSize uint64 `json:"metadata_size"` - RBACSize uint64 `json:"rbac_size,omitempty"` - ConfigSize uint64 `json:"config_size,omitempty"` - CompressedSize uint64 `json:"compressed_size,omitempty"` - Databases []DatabasesMeta `json:"databases,omitempty"` - Tables []TableTitle `json:"tables"` - Functions []FunctionsMeta `json:"functions"` - DataFormat string `json:"data_format"` - RequiredBackup string `json:"required_backup,omitempty"` -} - type DatabasesMeta struct { Name string `json:"name"` Engine string `json:"engine"` @@ -40,37 +16,15 @@ type FunctionsMeta struct { CreateQuery string `json:"create_query"` } -type TableMetadata struct { - Files map[string][]string `json:"files,omitempty"` - Table string `json:"table"` - Database string `json:"database"` - Parts map[string][]Part `json:"parts"` - Query string `json:"query"` - Size map[string]int64 `json:"size"` // how much size on each disk - TotalBytes uint64 `json:"total_bytes,omitempty"` // total table size - DependenciesTable string `json:"dependencies_table,omitempty"` - DependenciesDatabase string `json:"dependencies_database,omitempty"` - Mutations []MutationMetadata `json:"mutations,omitempty"` - MetadataOnly bool `json:"metadata_only"` -} - type MutationMetadata struct { MutationId string `json:"mutation_id" ch:"mutation_id"` Command string `json:"command" ch:"command"` } type Part struct { - Name string `json:"name"` - Required bool `json:"required,omitempty"` - Partition string `json:"partition,omitempty"` - // Path string `json:"path"` // TODO: make it relative? look like useless now, can be calculated from Name - HashOfAllFiles string `json:"hash_of_all_files,omitempty"` // ??? - HashOfUncompressedFiles string `json:"hash_of_uncompressed_files,omitempty"` - UncompressedHashOfCompressedFiles string `json:"uncompressed_hash_of_compressed_files,omitempty"` // ??? - PartitionID string `json:"partition_id,omitempty"` - ModificationTime *time.Time `json:"modification_time,omitempty"` - Size int64 `json:"size,omitempty"` - // bytes_on_disk, data_compressed_bytes, data_uncompressed_bytes + Name string `json:"name"` + Required bool `json:"required,omitempty"` + RebalancedDisk string `json:"rebalanced_disk,omitempty"` } type SplitPartFiles struct { diff --git a/pkg/metadata/save.go b/pkg/metadata/save.go deleted file mode 100644 index 7947f5cf..00000000 --- a/pkg/metadata/save.go +++ /dev/null @@ -1,46 +0,0 @@ -package metadata - -import ( - "encoding/json" - "fmt" - "os" - "path" -) - -func (tm *TableMetadata) Save(location string, metadataOnly bool) (uint64, error) { - newTM := TableMetadata{ - Table: tm.Table, - Database: tm.Database, - Query: tm.Query, - DependenciesTable: tm.DependenciesTable, - DependenciesDatabase: tm.DependenciesDatabase, - MetadataOnly: true, - } - - if !metadataOnly { - newTM.Files = tm.Files - newTM.Parts = tm.Parts - newTM.Size = tm.Size - newTM.TotalBytes = tm.TotalBytes - newTM.MetadataOnly = false - } - if err := os.MkdirAll(path.Dir(location), 0750); err != nil { - return 0, err - } - body, err := json.MarshalIndent(&newTM, "", "\t") - if err != nil { - return 0, err - } - return uint64(len(body)), os.WriteFile(location, body, 0640) -} - -func (bm *BackupMetadata) Save(location string) error { - tbBody, err := json.MarshalIndent(bm, "", "\t") - if err != nil { - return fmt.Errorf("can't marshall backup metadata: %v", err) - } - if err := os.WriteFile(location, tbBody, 0640); err != nil { - return fmt.Errorf("can't save backup metadata: %v", err) - } - return nil -} diff --git a/pkg/metadata/table_metadata.go b/pkg/metadata/table_metadata.go new file mode 100644 index 00000000..82bedf5d --- /dev/null +++ b/pkg/metadata/table_metadata.go @@ -0,0 +1,64 @@ +package metadata + +import ( + "encoding/json" + apexLog "github.com/apex/log" + "os" + "path" +) + +type TableMetadata struct { + Files map[string][]string `json:"files,omitempty"` + RebalancedFiles map[string]string `json:"rebalanced_files,omitempty"` + Table string `json:"table"` + Database string `json:"database"` + Parts map[string][]Part `json:"parts"` + Query string `json:"query"` + Size map[string]int64 `json:"size"` // how much size on each disk + TotalBytes uint64 `json:"total_bytes,omitempty"` // total table size + DependenciesTable string `json:"dependencies_table,omitempty"` + DependenciesDatabase string `json:"dependencies_database,omitempty"` + Mutations []MutationMetadata `json:"mutations,omitempty"` + MetadataOnly bool `json:"metadata_only"` + LocalFile string `json:"local_file,omitempty"` +} + +func (tm *TableMetadata) Save(location string, metadataOnly bool) (uint64, error) { + newTM := TableMetadata{ + Table: tm.Table, + Database: tm.Database, + Query: tm.Query, + DependenciesTable: tm.DependenciesTable, + DependenciesDatabase: tm.DependenciesDatabase, + MetadataOnly: true, + } + + if !metadataOnly { + newTM.Files = tm.Files + newTM.Parts = tm.Parts + newTM.Size = tm.Size + newTM.TotalBytes = tm.TotalBytes + newTM.MetadataOnly = false + } + if err := os.MkdirAll(path.Dir(location), 0750); err != nil { + return 0, err + } + body, err := json.MarshalIndent(&newTM, "", "\t") + if err != nil { + return 0, err + } + return uint64(len(body)), os.WriteFile(location, body, 0640) +} + +func (tm *TableMetadata) Load(location string) (uint64, error) { + log := apexLog.WithField("logger", "metadata.Load") + data, err := os.ReadFile(location) + if err != nil { + return 0, err + } + if err := json.Unmarshal(data, tm); err != nil { + return 0, err + } + log.Debugf("success %s", location) + return uint64(len(data)), nil +} diff --git a/pkg/partition/partition.go b/pkg/partition/partition.go index 783debde..eda15102 100644 --- a/pkg/partition/partition.go +++ b/pkg/partition/partition.go @@ -3,12 +3,15 @@ package partition import ( "context" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/common" - "github.com/Altinity/clickhouse-backup/pkg/metadata" - apexLog "github.com/apex/log" + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/common" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + "path/filepath" "regexp" + "sort" "strconv" "strings" ) @@ -48,7 +51,7 @@ var SettingsRE = regexp.MustCompile(`(?mi)\s*SETTINGS.*`) var OrderByRE = regexp.MustCompile(`(?mi)\s*ORDER BY.*`) var FunctionsRE = regexp.MustCompile(`(?i)\w+\(`) var StringsRE = regexp.MustCompile(`(?i)'[^']+'`) -var SpecialCharsRE = regexp.MustCompile(`(?i)[)*+\-/\\]+`) +var SpecialCharsRE = regexp.MustCompile(`(?i)[)(*+\-/\\,]+`) var FieldsNamesRE = regexp.MustCompile("(?i)\\w+|`[^`]+`\\.`[^`]+`|\"[^\"]+\"") func extractPartitionByFieldNames(s string) []struct { @@ -59,7 +62,7 @@ func extractPartitionByFieldNames(s string) []struct { s = FunctionsRE.ReplaceAllString(s, "") s = StringsRE.ReplaceAllString(s, "") s = SpecialCharsRE.ReplaceAllString(s, "") - matches := FieldsNamesRE.FindStringSubmatch(s) + matches := FieldsNamesRE.FindAllString(s, -1) columns := make([]struct { Name string `ch:"name"` }, len(matches)) @@ -95,24 +98,30 @@ func GetPartitionIdAndName(ctx context.Context, ch *clickhouse.ClickHouse, datab }, 0) sql := "SELECT name FROM system.columns WHERE database=? AND table=? AND is_in_partition_key" oldVersion := false + partitionByMatches := PartitionByRE.FindStringSubmatch(createQuery) if err := ch.SelectContext(ctx, &columns, sql, database, partitionIdTable); err != nil { - matches := PartitionByRE.FindStringSubmatch(createQuery) - if len(matches) == 0 { + if len(partitionByMatches) == 0 { if dropErr := dropPartitionIdTable(ch, database, partitionIdTable); dropErr != nil { return "", "", dropErr } return "", "", fmt.Errorf("can't get is_in_partition_key column names from for table `%s`.`%s`: %v", database, partitionIdTable, err) } - columns = extractPartitionByFieldNames(matches[1]) + columns = extractPartitionByFieldNames(partitionByMatches[1]) oldVersion = true } + // to the same order of fields as described in PARTITION BY clause, https://github.com/Altinity/clickhouse-backup/issues/791 + if len(partitionByMatches) == 2 && partitionByMatches[1] != "" { + sort.Slice(columns, func(i int, j int) bool { + return strings.Index(partitionByMatches[1], columns[i].Name) < strings.Index(partitionByMatches[1], columns[j].Name) + }) + } defer func() { if dropErr := dropPartitionIdTable(ch, database, partitionIdTable); dropErr != nil { - apexLog.Warnf("partition.GetPartitionId can't drop `%s`.`%s`: %v", database, partitionIdTable, dropErr) + log.Warn().Msgf("partition.GetPartitionId can't drop `%s`.`%s`: %v", database, partitionIdTable, dropErr) } }() if len(columns) == 0 { - apexLog.Warnf("is_in_partition_key=1 fields not found in system.columns for table `%s`.`%s`", database, partitionIdTable) + log.Warn().Msgf("is_in_partition_key=1 fields not found in system.columns for table `%s`.`%s`", database, partitionIdTable) return "", "", nil } partitionInsert := splitAndParsePartition(partition) @@ -135,13 +144,13 @@ func GetPartitionIdAndName(ctx context.Context, ch *clickhouse.ClickHouse, datab ) batch, err := ch.GetConn().PrepareBatch(ctx, sql) if err != nil { - return "", "", err + return "", "", errors.Wrapf(err, "PrepareBatch sql=%s partitionInsert=%#v", sql, partitionInsert) } if err = batch.Append(partitionInsert...); err != nil { - return "", "", err + return "", "", errors.Wrapf(err, "batch.Append sql=%s partitionInsert=%#v", sql, partitionInsert) } if err = batch.Send(); err != nil { - return "", "", err + return "", "", errors.Wrapf(err, "batch.Send sql=%s partitionInsert=%#v", sql, partitionInsert) } } if err != nil { @@ -194,6 +203,12 @@ func ConvertPartitionsToIdsMapAndNamesList(ctx context.Context, ch *clickhouse.C // to allow use --partitions val1 --partitions val2, https://github.com/Altinity/clickhouse-backup/issues/425#issuecomment-1149855063 for _, partitionArg := range partitions { partitionArg = strings.Trim(partitionArg, " \t") + tablePattern := "*.*" + // when PARTITION BY is table specific, https://github.com/Altinity/clickhouse-backup/issues/916 + if tablePatternDelimiterIndex := strings.Index(partitionArg, ":"); tablePatternDelimiterIndex != -1 { + tablePattern = partitionArg[:tablePatternDelimiterIndex] + partitionArg = strings.TrimPrefix(partitionArg, tablePattern+":") + } // when PARTITION BY clause return partition_id field as hash, https://github.com/Altinity/clickhouse-backup/issues/602 if strings.HasPrefix(partitionArg, "(") { partitionArg = strings.TrimSuffix(strings.TrimPrefix(partitionArg, "("), ")") @@ -201,17 +216,17 @@ func ConvertPartitionsToIdsMapAndNamesList(ctx context.Context, ch *clickhouse.C for _, t := range tablesFromClickHouse { createIdMapAndNameListIfNotExists(t.Database, t.Name, partitionsIdMap, partitionsNameList) if partitionId, partitionName, err := GetPartitionIdAndName(ctx, ch, t.Database, t.Name, t.CreateTableQuery, partitionTuple); err != nil { - apexLog.Fatalf("partition.GetPartitionIdAndName error: %v", err) + log.Fatal().Msgf("partition.GetPartitionIdAndName error: %v", err) } else if partitionId != "" { - addItemToIdMapAndNameListIfNotExists(partitionId, partitionName, t.Database, t.Name, partitionsIdMap, partitionsNameList) + addItemToIdMapAndNameListIfNotExists(partitionId, partitionName, t.Database, t.Name, partitionsIdMap, partitionsNameList, tablePattern) } } for _, t := range tablesFromMetadata { createIdMapAndNameListIfNotExists(t.Database, t.Table, partitionsIdMap, partitionsNameList) if partitionId, partitionName, err := GetPartitionIdAndName(ctx, ch, t.Database, t.Table, t.Query, partitionTuple); err != nil { - apexLog.Fatalf("partition.GetPartitionIdAndName error: %v", err) + log.Fatal().Msgf("partition.GetPartitionIdAndName error: %v", err) } else if partitionId != "" { - addItemToIdMapAndNameListIfNotExists(partitionId, partitionName, t.Database, t.Table, partitionsIdMap, partitionsNameList) + addItemToIdMapAndNameListIfNotExists(partitionId, partitionName, t.Database, t.Table, partitionsIdMap, partitionsNameList, tablePattern) } } } @@ -221,11 +236,11 @@ func ConvertPartitionsToIdsMapAndNamesList(ctx context.Context, ch *clickhouse.C item = strings.Trim(item, " \t") for _, t := range tablesFromClickHouse { createIdMapAndNameListIfNotExists(t.Database, t.Name, partitionsIdMap, partitionsNameList) - addItemToIdMapAndNameListIfNotExists(item, item, t.Database, t.Name, partitionsIdMap, partitionsNameList) + addItemToIdMapAndNameListIfNotExists(item, item, t.Database, t.Name, partitionsIdMap, partitionsNameList, tablePattern) } for _, t := range tablesFromMetadata { createIdMapAndNameListIfNotExists(t.Database, t.Table, partitionsIdMap, partitionsNameList) - addItemToIdMapAndNameListIfNotExists(item, item, t.Database, t.Table, partitionsIdMap, partitionsNameList) + addItemToIdMapAndNameListIfNotExists(item, item, t.Database, t.Table, partitionsIdMap, partitionsNameList, tablePattern) } } } @@ -233,24 +248,28 @@ func ConvertPartitionsToIdsMapAndNamesList(ctx context.Context, ch *clickhouse.C return partitionsIdMap, partitionsNameList } -func addItemToIdMapAndNameListIfNotExists(partitionId, partitionName, database string, table string, partitionsIdMap map[metadata.TableTitle]common.EmptyMap, partitionsNameList map[metadata.TableTitle][]string) { - if partitionId != "" { - partitionsIdMap[metadata.TableTitle{ - Database: database, Table: table, - }][partitionId] = struct{}{} - } - if partitionName != "" { - partitionsNameList[metadata.TableTitle{ - Database: database, Table: table, - }] = common.AddStringToSliceIfNotExists(partitionsNameList[metadata.TableTitle{ - Database: database, Table: table, - }], partitionName) +func addItemToIdMapAndNameListIfNotExists(partitionId, partitionName, database, table string, partitionsIdMap map[metadata.TableTitle]common.EmptyMap, partitionsNameList map[metadata.TableTitle][]string, tablePattern string) { + if matched, err := filepath.Match(tablePattern, database+"."+table); err == nil && matched { + if partitionId != "" { + partitionsIdMap[metadata.TableTitle{ + Database: database, Table: table, + }][partitionId] = struct{}{} + } + if partitionName != "" { + partitionsNameList[metadata.TableTitle{ + Database: database, Table: table, + }] = common.AddStringToSliceIfNotExists(partitionsNameList[metadata.TableTitle{ + Database: database, Table: table, + }], partitionName) + } + } else if err != nil { + log.Error().Msgf("wrong --partitions table specific pattern matching: %v", err) } } func createIdMapAndNameListIfNotExists(database, table string, partitionsIdsMap map[metadata.TableTitle]common.EmptyMap, partitionsNameList map[metadata.TableTitle][]string) { if _, exists := partitionsIdsMap[metadata.TableTitle{Database: database, Table: table}]; !exists { - partitionsIdsMap[metadata.TableTitle{Database: database, Table: table}] = make(common.EmptyMap, 0) + partitionsIdsMap[metadata.TableTitle{Database: database, Table: table}] = make(common.EmptyMap) } if _, exists := partitionsNameList[metadata.TableTitle{Database: database, Table: table}]; !exists { partitionsNameList[metadata.TableTitle{Database: database, Table: table}] = make([]string, 0) diff --git a/pkg/progressbar/progressbar.go b/pkg/progressbar/progressbar.go deleted file mode 100644 index 696aabbe..00000000 --- a/pkg/progressbar/progressbar.go +++ /dev/null @@ -1,57 +0,0 @@ -package progressbar - -import ( - "fmt" - "io" - - progressbar "gopkg.in/cheggaaa/pb.v1" -) - -type Bar struct { - pb *progressbar.ProgressBar - show bool -} - -func StartNewByteBar(show bool, total int64) *Bar { - if show { - return &Bar{ - show: true, - pb: progressbar.StartNew(int(total)).SetUnits(progressbar.U_BYTES), - } - } - return &Bar{ - show: false, - } -} - -func (b *Bar) Finish() { - if b.show { - b.pb.Finish() - fmt.Print("\033[A") // move the cursor up - } -} - -func (b *Bar) Add64(add int64) { - if b.show { - b.pb.Add64(add) - } -} - -func (b *Bar) Set(current int) { - if b.show { - b.pb.Set(current) - } -} - -func (b *Bar) Increment() { - if b.show { - b.pb.Increment() - } -} - -func (b *Bar) NewProxyReader(r io.Reader) io.Reader { - if b.show { - return b.pb.NewProxyReader(r) - } - return r -} diff --git a/pkg/server/callback_test.go b/pkg/server/callback_test.go index adf57c95..2e8fdcd1 100644 --- a/pkg/server/callback_test.go +++ b/pkg/server/callback_test.go @@ -28,9 +28,11 @@ func TestParseCallback(t *testing.T) { goodChan2 := make(chan *payload, 5) passToChanHandler := func(ch chan *payload) http.HandlerFunc { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { defer func() { - _ = r.Body.Close() + if err := r.Body.Close(); err != nil { + t.Fatalf("can't close r.Body: %v", err) + } }() var data payload @@ -41,7 +43,7 @@ func TestParseCallback(t *testing.T) { if _, err := w.Write(nil); err != nil { t.Fatalf("unexpected error while writing response from test server: %v", err) } - }) + } } returnErrHandler := http.HandlerFunc( func(w http.ResponseWriter, _ *http.Request) { diff --git a/pkg/server/metrics/metrics.go b/pkg/server/metrics/metrics.go index ecbca8bf..597886a6 100644 --- a/pkg/server/metrics/metrics.go +++ b/pkg/server/metrics/metrics.go @@ -31,6 +31,7 @@ type APIMetrics struct { NumberBackupsLocal prometheus.Gauge NumberBackupsRemoteExpected prometheus.Gauge NumberBackupsLocalExpected prometheus.Gauge + InProgressCommands prometheus.Gauge SubCommands map[string][]string } @@ -100,6 +101,7 @@ func (m *APIMetrics) RegisterMetrics() { Name: "last_backup_size_local", Help: "Last local backup size in bytes", }) + m.LastBackupSizeRemote = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "clickhouse_backup", Name: "last_backup_size_remote", @@ -136,6 +138,12 @@ func (m *APIMetrics) RegisterMetrics() { Help: "How many backups expected on local storage", }) + m.InProgressCommands = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "clickhouse_backup", + Name: "in_progress_commands", + Help: "How many commands running in progress", + }) + for _, command := range commandList { prometheus.MustRegister( m.SuccessfulCounter[command], @@ -154,6 +162,7 @@ func (m *APIMetrics) RegisterMetrics() { m.NumberBackupsLocal, m.NumberBackupsRemoteExpected, m.NumberBackupsLocalExpected, + m.InProgressCommands, ) for _, command := range commandList { @@ -164,6 +173,7 @@ func (m *APIMetrics) RegisterMetrics() { func (m *APIMetrics) Start(command string, startTime time.Time) { if _, exists := m.LastStart[command]; exists { m.LastStart[command].Set(float64(startTime.Unix())) + m.InProgressCommands.Inc() if subCommands, subCommandsExists := m.SubCommands[command]; subCommandsExists { for _, subCommand := range subCommands { if _, exists := m.LastStart[subCommand]; exists { @@ -175,10 +185,12 @@ func (m *APIMetrics) Start(command string, startTime time.Time) { log.Warn().Msgf("%s not found in LastStart metrics", command) } } + func (m *APIMetrics) Finish(command string, startTime time.Time) { if _, exists := m.LastFinish[command]; exists { m.LastDuration[command].Set(float64(time.Since(startTime).Nanoseconds())) m.LastFinish[command].Set(float64(time.Now().Unix())) + m.InProgressCommands.Dec() if subCommands, subCommandsExists := m.SubCommands[command]; subCommandsExists { for _, subCommand := range subCommands { if _, exists := m.LastFinish[subCommand]; exists { diff --git a/pkg/server/server.go b/pkg/server/server.go index 16b63e31..11cd3257 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -22,19 +22,20 @@ import ( "syscall" "time" - "github.com/Altinity/clickhouse-backup/pkg/backup" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/common" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/resumable" - "github.com/Altinity/clickhouse-backup/pkg/server/metrics" - "github.com/Altinity/clickhouse-backup/pkg/status" - "github.com/Altinity/clickhouse-backup/pkg/utils" "github.com/google/shlex" "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog/log" "github.com/urfave/cli" + + "github.com/Altinity/clickhouse-backup/v2/pkg/backup" + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/common" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/resumable" + "github.com/Altinity/clickhouse-backup/v2/pkg/server/metrics" + "github.com/Altinity/clickhouse-backup/v2/pkg/status" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" ) type APIServer struct { @@ -44,6 +45,7 @@ type APIServer struct { config *config.Config server *http.Server restart chan struct{} + stop chan struct{} metrics *metrics.APIMetrics routes []string clickhouseBackupVersion string @@ -63,7 +65,7 @@ func Run(cliCtx *cli.Context, cliApp *cli.App, configPath string, clickhouseBack for { cfg, err = config.LoadConfig(configPath) if err != nil { - log.Error().Err(err).Send() + log.Error().Stack().Err(err).Send() time.Sleep(5 * time.Second) continue } @@ -71,7 +73,7 @@ func Run(cliCtx *cli.Context, cliApp *cli.App, configPath string, clickhouseBack Config: &cfg.ClickHouse, } if err := ch.Connect(); err != nil { - log.Error().Err(err).Send() + log.Error().Stack().Err(err).Send() time.Sleep(5 * time.Second) continue } @@ -86,6 +88,7 @@ func Run(cliCtx *cli.Context, cliApp *cli.App, configPath string, clickhouseBack restart: make(chan struct{}), clickhouseBackupVersion: clickhouseBackupVersion, metrics: metrics.NewAPIMetrics(), + stop: make(chan struct{}), } if cfg.API.CreateIntegrationTables { if err := api.CreateIntegrationTables(); err != nil { @@ -94,7 +97,7 @@ func Run(cliCtx *cli.Context, cliApp *cli.App, configPath string, clickhouseBack } api.metrics.RegisterMetrics() - log.Info().Msgf("Starting API server on %s", api.config.API.ListenAddr) + log.Info().Msgf("Starting API server %s on %s", api.cliApp.Version, api.config.API.ListenAddr) sigterm := make(chan os.Signal, 1) signal.Notify(sigterm, os.Interrupt, syscall.SIGTERM) sighup := make(chan os.Signal, 1) @@ -111,8 +114,8 @@ func Run(cliCtx *cli.Context, cliApp *cli.App, configPath string, clickhouseBack } go func() { - if err := api.UpdateBackupMetrics(context.Background(), false); err != nil { - log.Error().Msgf("UpdateBackupMetrics return error: %v", err) + if metricsErr := api.UpdateBackupMetrics(context.Background(), false); metricsErr != nil { + log.Error().Msgf("UpdateBackupMetrics return error: %v", metricsErr) } }() @@ -137,6 +140,9 @@ func Run(cliCtx *cli.Context, cliApp *cli.App, configPath string, clickhouseBack case <-sigterm: log.Info().Msg("Stopping API server") return api.Stop() + case <-api.stop: + log.Info().Msg("Stopping API server. Stopped from the inside of the application") + return api.Stop() } } } @@ -154,7 +160,7 @@ func (api *APIServer) RunWatch(cliCtx *cli.Context) { "*.*", nil, false, false, false, false, api.clickhouseBackupVersion, commandId, api.GetMetrics(), cliCtx, ) - status.Current.Stop(commandId, err) + api.handleWatchResponse(commandId, err) } // Stop cancel all running commands, @todo think about graceful period @@ -214,6 +220,7 @@ func (api *APIServer) registerHTTPHandlers() *http.Server { r.HandleFunc("/", api.httpRootHandler).Methods("GET", "HEAD") r.HandleFunc("/", api.httpRestartHandler).Methods("POST") r.HandleFunc("/restart", api.httpRestartHandler).Methods("POST", "GET") + r.HandleFunc("/backup/version", api.httpVersionHandler).Methods("GET", "HEAD") r.HandleFunc("/backup/kill", api.httpKillHandler).Methods("POST", "GET") r.HandleFunc("/backup/watch", api.httpWatchHandler).Methods("POST", "GET") r.HandleFunc("/backup/tables", api.httpTablesHandler).Methods("GET") @@ -324,7 +331,7 @@ func (api *APIServer) actions(w http.ResponseWriter, r *http.Request) { api.writeError(w, http.StatusBadRequest, string(line), err) return } - log.Info().Msgf("/backup/actions call: %s", row.Command) + log.Info().Str("version", api.cliApp.Version).Msgf("/backup/actions call: %s", row.Command) args, err := shlex.Split(row.Command) if err != nil { api.writeError(w, http.StatusBadRequest, "", err) @@ -339,6 +346,12 @@ func (api *APIServer) actions(w http.ResponseWriter, r *http.Request) { api.writeError(w, http.StatusInternalServerError, row.Command, err) return } + case "clean": + actionsResults, err = api.actionsCleanHandler(w, row, command, actionsResults) + if err != nil { + api.writeError(w, http.StatusInternalServerError, row.Command, err) + return + } case "clean_remote_broken": actionsResults, err = api.actionsCleanRemoteBrokenHandler(w, row, command, actionsResults) if err != nil { @@ -351,7 +364,7 @@ func (api *APIServer) actions(w http.ResponseWriter, r *http.Request) { api.writeError(w, http.StatusInternalServerError, row.Command, err) return } - case "create", "restore", "upload", "download", "create_remote", "restore_remote": + case "create", "restore", "upload", "download", "create_remote", "restore_remote", "list": actionsResults, err = api.actionsAsyncCommandsHandler(command, args, row, actionsResults) if err != nil { api.writeError(w, http.StatusInternalServerError, row.Command, err) @@ -382,8 +395,8 @@ func (api *APIServer) actionsDeleteHandler(row status.ActionRow, args []string, return actionsResults, err } go func() { - if err := api.UpdateBackupMetrics(context.Background(), args[1] == "local"); err != nil { - log.Error().Msgf("UpdateBackupMetrics return error: %v", err) + if metricsErr := api.UpdateBackupMetrics(context.Background(), args[1] == "local"); metricsErr != nil { + log.Error().Msgf("UpdateBackupMetrics return error: %v", metricsErr) } }() actionsResults = append(actionsResults, actionsResultsRow{ @@ -410,7 +423,7 @@ func (api *APIServer) actionsAsyncCommandsHandler(command string, args []string, return } go func() { - if err := api.UpdateBackupMetrics(context.Background(), command == "create" || command == "restore"); err != nil { + if err := api.UpdateBackupMetrics(context.Background(), command == "create" || strings.HasPrefix(command, "restore") || command == "download"); err != nil { log.Error().Msgf("UpdateBackupMetrics return error: %v", err) } }() @@ -423,10 +436,10 @@ func (api *APIServer) actionsAsyncCommandsHandler(command string, args []string, } func (api *APIServer) actionsKillHandler(row status.ActionRow, args []string, actionsResults []actionsResultsRow) ([]actionsResultsRow, error) { - if len(args) <= 1 { - return actionsResults, errors.New("kill parameter empty") + killCommand := "" + if len(args) > 1 { + killCommand = args[1] } - killCommand := args[1] commandId, _ := status.Current.Start(row.Command) err := status.Current.Cancel(killCommand, fmt.Errorf("canceled from API /backup/actions")) defer status.Current.Stop(commandId, err) @@ -440,12 +453,44 @@ func (api *APIServer) actionsKillHandler(row status.ActionRow, args []string, ac return actionsResults, nil } +func (api *APIServer) actionsCleanHandler(w http.ResponseWriter, row status.ActionRow, command string, actionsResults []actionsResultsRow) ([]actionsResultsRow, error) { + if !api.config.API.AllowParallel && status.Current.InProgress() { + log.Warn().Msgf(ErrAPILocked.Error()) + return actionsResults, ErrAPILocked + } + commandId, ctx := status.Current.Start(command) + cfg, err := api.ReloadConfig(w, "clean") + if err != nil { + status.Current.Stop(commandId, err) + return actionsResults, err + } + b := backup.NewBackuper(cfg) + err = b.Clean(ctx) + if err != nil { + log.Error().Msgf("actions Clean error: %v", err) + status.Current.Stop(commandId, err) + return actionsResults, err + } + log.Info().Msg("CLEANED") + go func() { + if metricsErr := api.UpdateBackupMetrics(context.Background(), true); metricsErr != nil { + log.Error().Msgf("UpdateBackupMetrics return error: %v", metricsErr) + } + }() + status.Current.Stop(commandId, nil) + actionsResults = append(actionsResults, actionsResultsRow{ + Status: "success", + Operation: row.Command, + }) + return actionsResults, nil +} + func (api *APIServer) actionsCleanRemoteBrokenHandler(w http.ResponseWriter, row status.ActionRow, command string, actionsResults []actionsResultsRow) ([]actionsResultsRow, error) { if !api.config.API.AllowParallel && status.Current.InProgress() { log.Warn().Err(ErrAPILocked).Send() return actionsResults, ErrAPILocked } - commandId, ctx := status.Current.Start(command) + commandId, _ := status.Current.Start(command) cfg, err := api.ReloadConfig(w, "clean_remote_broken") if err != nil { status.Current.Stop(commandId, err) @@ -459,10 +504,11 @@ func (api *APIServer) actionsCleanRemoteBrokenHandler(w http.ResponseWriter, row return actionsResults, err } log.Info().Msg("CLEANED") - metricsErr := api.UpdateBackupMetrics(ctx, false) - if metricsErr != nil { - log.Error().Msgf("UpdateBackupMetrics return error: %v", metricsErr) - } + go func() { + if metricsErr := api.UpdateBackupMetrics(context.Background(), false); metricsErr != nil { + log.Error().Msgf("UpdateBackupMetrics return error: %v", metricsErr) + } + }() status.Current.Stop(commandId, nil) actionsResults = append(actionsResults, actionsResultsRow{ Status: "success", @@ -524,7 +570,7 @@ func (api *APIServer) actionsWatchHandler(w http.ResponseWriter, row status.Acti fullCommand = fmt.Sprintf("%s --tables=\"%s\"", fullCommand, tablePattern) } if matchParam, partitions := simpleParseArg(i, args, "--partitions"); matchParam { - partitionsToBackup = strings.Split(partitions, ",") + partitionsToBackup = append(partitionsToBackup, partitions) fullCommand = fmt.Sprintf("%s --partitions=\"%s\"", fullCommand, partitions) } if matchParam, _ = simpleParseArg(i, args, "--schema"); matchParam { @@ -549,11 +595,7 @@ func (api *APIServer) actionsWatchHandler(w http.ResponseWriter, row status.Acti go func() { b := backup.NewBackuper(cfg) err := b.Watch(watchInterval, fullInterval, watchBackupNameTemplate, tablePattern, partitionsToBackup, schemaOnly, rbacOnly, configsOnly, skipCheckPartsColumns, api.clickhouseBackupVersion, commandId, api.GetMetrics(), api.cliCtx) - defer status.Current.Stop(commandId, err) - if err != nil { - log.Error().Msgf("Watch error: %v", err) - return - } + api.handleWatchResponse(commandId, err) }() actionsResults = append(actionsResults, actionsResultsRow{ @@ -563,6 +605,21 @@ func (api *APIServer) actionsWatchHandler(w http.ResponseWriter, row status.Acti return actionsResults, nil } +func (api *APIServer) handleWatchResponse(watchCommandId int, err error) { + status.Current.Stop(watchCommandId, err) + if err != nil { + log.Error().Msgf("Watch error: %v", err) + } + if api.config.API.WatchIsMainProcess { + // Do not stop server if 'watch' was canceled by the user command + if errors.Is(err, context.Canceled) { + return + } + log.Info().Msg("Stopping server since watch command is stopped") + api.stop <- struct{}{} + } +} + func (api *APIServer) actionsLog(w http.ResponseWriter, r *http.Request) { var last int64 var err error @@ -588,7 +645,7 @@ func (api *APIServer) httpRootHandler(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Cache-Control", "no-store, no-cache, must-revalidate") w.Header().Set("Pragma", "no-cache") - _, _ = fmt.Fprintln(w, "Documentation: https://github.com/Altinity/clickhouse-backup#api") + _, _ = fmt.Fprintf(w, "Version: %s\nDocumentation: https://github.com/Altinity/clickhouse-backup#api\n", api.cliApp.Version) for _, r := range api.routes { _, _ = fmt.Fprintln(w, r) } @@ -608,6 +665,15 @@ func (api *APIServer) httpRestartHandler(w http.ResponseWriter, _ *http.Request) }() } +// httpVersionHandler +func (api *APIServer) httpVersionHandler(w http.ResponseWriter, _ *http.Request) { + api.sendJSONEachRow(w, http.StatusOK, struct { + Version string `json:"version"` + }{ + Version: api.cliApp.Version, + }) +} + // httpKillHandler - kill selected command if it InProgress func (api *APIServer) httpKillHandler(w http.ResponseWriter, r *http.Request) { var err error @@ -615,7 +681,7 @@ func (api *APIServer) httpKillHandler(w http.ResponseWriter, r *http.Request) { if exists && len(command) > 0 { err = status.Current.Cancel(command[0], fmt.Errorf("canceled from API /backup/kill")) } else { - err = fmt.Errorf("require non empty `command` parameter") + err = status.Current.Cancel("", fmt.Errorf("canceled from API /backup/kill")) } if err != nil { api.sendJSONEachRow(w, http.StatusInternalServerError, struct { @@ -648,16 +714,22 @@ func (api *APIServer) httpTablesHandler(w http.ResponseWriter, r *http.Request) } b := backup.NewBackuper(cfg) q := r.URL.Query() - tables, err := b.GetTables(context.Background(), q.Get("table")) + var tables []clickhouse.Table + // https://github.com/Altinity/clickhouse-backup/issues/778 + if q.Get("remote_backup") != "" { + tables, err = b.GetTablesRemote(context.Background(), q.Get("remote_backup"), q.Get("table")) + } else { + tables, err = b.GetTables(context.Background(), q.Get("table")) + } if err != nil { api.writeError(w, http.StatusInternalServerError, "tables", err) return } - if r.URL.Path != "/backup/tables/all" { - tables := api.getTablesWithSkip(tables) + if r.URL.Path == "/backup/tables/all" { api.sendJSONEachRow(w, http.StatusOK, tables) return } + tables = api.getTablesWithSkip(tables) api.sendJSONEachRow(w, http.StatusOK, tables) } @@ -723,9 +795,6 @@ func (api *APIServer) httpListHandler(w http.ResponseWriter, r *http.Request) { } for _, item := range localBackups { description := item.DataFormat - if item.Legacy { - description = "old-format" - } if item.Broken != "" { description = item.Broken } @@ -738,7 +807,7 @@ func (api *APIServer) httpListHandler(w http.ResponseWriter, r *http.Request) { backupsJSON = append(backupsJSON, backupJSON{ Name: item.BackupName, Created: item.CreationDate.Format(common.TimeFormat), - Size: item.DataSize + item.MetadataSize, + Size: item.GetFullSize(), Location: "local", RequiredBackup: item.RequiredBackup, Desc: description, @@ -755,9 +824,6 @@ func (api *APIServer) httpListHandler(w http.ResponseWriter, r *http.Request) { } for i, b := range remoteBackups { description := b.DataFormat - if b.Legacy { - description = "old-format" - } if b.Broken != "" { description = b.Broken brokenBackups++ @@ -768,16 +834,17 @@ func (api *APIServer) httpListHandler(w http.ResponseWriter, r *http.Request) { } description += b.Tags } + fullSize := b.GetFullSize() backupsJSON = append(backupsJSON, backupJSON{ Name: b.BackupName, Created: b.CreationDate.Format(common.TimeFormat), - Size: b.DataSize + b.MetadataSize, + Size: fullSize, Location: "remote", RequiredBackup: b.RequiredBackup, Desc: description, }) if i == len(remoteBackups)-1 { - api.metrics.LastBackupSizeRemote.Set(float64(b.DataSize + b.MetadataSize + b.ConfigSize + b.RBACSize)) + api.metrics.LastBackupSizeRemote.Set(float64(fullSize)) } } api.metrics.NumberBackupsRemoteBroken.Set(float64(brokenBackups)) @@ -798,6 +865,7 @@ func (api *APIServer) httpCreateHandler(w http.ResponseWriter, r *http.Request) return } tablePattern := "" + diffFromRemote := "" partitionsToBackup := make([]string, 0) backupName := backup.NewBackupName() schemaOnly := false @@ -810,9 +878,12 @@ func (api *APIServer) httpCreateHandler(w http.ResponseWriter, r *http.Request) tablePattern = tp[0] fullCommand = fmt.Sprintf("%s --tables=\"%s\"", fullCommand, tablePattern) } + if baseBackup, exists := query["diff-from-remote"]; exists { + diffFromRemote = baseBackup[0] + } if partitions, exist := query["partitions"]; exist { - partitionsToBackup = strings.Split(partitions[0], ",") - fullCommand = fmt.Sprintf("%s --partitions=\"%s\"", fullCommand, partitions) + partitionsToBackup = append(partitionsToBackup, partitions...) + fullCommand = fmt.Sprintf("%s --partitions=\"%s\"", fullCommand, strings.Join(partitions, "\" --partitions=\"")) } if schema, exist := query["schema"]; exist { schemaOnly, _ = strconv.ParseBool(schema[0]) @@ -850,11 +921,11 @@ func (api *APIServer) httpCreateHandler(w http.ResponseWriter, r *http.Request) return } - commandId, ctx := status.Current.Start(fullCommand) + commandId, _ := status.Current.Start(fullCommand) go func() { err, _ := api.metrics.ExecuteWithMetrics("create", 0, func() error { b := backup.NewBackuper(cfg) - return b.CreateBackup(backupName, tablePattern, partitionsToBackup, schemaOnly, createRBAC, false, createConfigs, false, checkPartsColumns, api.clickhouseBackupVersion, commandId) + return b.CreateBackup(backupName, diffFromRemote, tablePattern, partitionsToBackup, schemaOnly, createRBAC, false, createConfigs, false, checkPartsColumns, api.clickhouseBackupVersion, commandId) }) if err != nil { log.Error().Msgf("API /backup/create error: %v", err) @@ -862,12 +933,12 @@ func (api *APIServer) httpCreateHandler(w http.ResponseWriter, r *http.Request) api.errorCallback(context.Background(), err, callback) return } - if err := api.UpdateBackupMetrics(ctx, true); err != nil { - log.Error().Msgf("UpdateBackupMetrics return error: %v", err) - status.Current.Stop(commandId, err) - api.errorCallback(context.Background(), err, callback) - return - } + go func() { + if metricsErr := api.UpdateBackupMetrics(context.Background(), true); metricsErr != nil { + log.Error().Msgf("UpdateBackupMetrics return error: %v", metricsErr) + } + }() + status.Current.Stop(commandId, nil) api.successCallback(context.Background(), callback) }() @@ -921,8 +992,8 @@ func (api *APIServer) httpWatchHandler(w http.ResponseWriter, r *http.Request) { fullCommand = fmt.Sprintf("%s --tables=\"%s\"", fullCommand, tablePattern) } if partitions, exist := query["partitions"]; exist { - partitionsToBackup = strings.Split(partitions[0], ",") - fullCommand = fmt.Sprintf("%s --partitions=\"%s\"", fullCommand, partitions) + partitionsToBackup = append(partitionsToBackup, partitions...) + fullCommand = fmt.Sprintf("%s --partitions=\"%s\"", fullCommand, strings.Join(partitions, "\" --partitions=\"")) } if schema, exist := query["schema"]; exist { schemaOnly, _ = strconv.ParseBool(schema[0]) @@ -959,11 +1030,7 @@ func (api *APIServer) httpWatchHandler(w http.ResponseWriter, r *http.Request) { go func() { b := backup.NewBackuper(cfg) err := b.Watch(watchInterval, fullInterval, watchBackupNameTemplate, tablePattern, partitionsToBackup, schemaOnly, rbacOnly, configsOnly, skipCheckPartsColumns, api.clickhouseBackupVersion, commandId, api.GetMetrics(), api.cliCtx) - defer status.Current.Stop(commandId, err) - if err != nil { - log.Error().Msgf("Watch error: %v", err) - return - } + api.handleWatchResponse(commandId, err) }() api.sendJSONEachRow(w, http.StatusCreated, struct { Status string `json:"status"` @@ -1004,7 +1071,7 @@ func (api *APIServer) httpCleanRemoteBrokenHandler(w http.ResponseWriter, _ *htt if err != nil { return } - commandId, ctx := status.Current.Start("clean_remote_broken") + commandId, _ := status.Current.Start("clean_remote_broken") defer status.Current.Stop(commandId, err) b := backup.NewBackuper(cfg) @@ -1014,13 +1081,11 @@ func (api *APIServer) httpCleanRemoteBrokenHandler(w http.ResponseWriter, _ *htt api.writeError(w, http.StatusInternalServerError, "clean_remote_broken", err) return } - - err = api.UpdateBackupMetrics(ctx, false) - if err != nil { - log.Error().Msgf("Clean remote broken error: %v", err) - api.writeError(w, http.StatusInternalServerError, "clean_remote_broken", err) - return - } + go func() { + if metricsErr := api.UpdateBackupMetrics(context.Background(), false); metricsErr != nil { + log.Error().Msgf("UpdateBackupMetrics return error: %v", metricsErr) + } + }() api.sendJSONEachRow(w, http.StatusOK, struct { Status string `json:"status"` @@ -1044,6 +1109,7 @@ func (api *APIServer) httpUploadHandler(w http.ResponseWriter, r *http.Request) } vars := mux.Vars(r) query := r.URL.Query() + deleteSource := false diffFrom := "" diffFromRemote := "" name := utils.CleanBackupNameRE.ReplaceAllString(vars["name"], "") @@ -1053,6 +1119,11 @@ func (api *APIServer) httpUploadHandler(w http.ResponseWriter, r *http.Request) resume := false fullCommand := "upload" + if _, exist := query["delete-source"]; exist { + deleteSource = true + fullCommand = fmt.Sprintf("%s --deleteSource", fullCommand) + } + if df, exist := query["diff-from"]; exist { diffFrom = df[0] fullCommand = fmt.Sprintf("%s --diff-from=\"%s\"", fullCommand, diffFrom) @@ -1066,8 +1137,8 @@ func (api *APIServer) httpUploadHandler(w http.ResponseWriter, r *http.Request) fullCommand = fmt.Sprintf("%s --tables=\"%s\"", fullCommand, tablePattern) } if partitions, exist := query["partitions"]; exist { - partitionsToBackup = strings.Split(partitions[0], ",") - fullCommand = fmt.Sprintf("%s --partitions=\"%s\"", fullCommand, partitions) + partitionsToBackup = append(partitionsToBackup, partitions...) + fullCommand = fmt.Sprintf("%s --partitions=\"%s\"", fullCommand, strings.Join(partitions, "\" --partitions=\"")) } if _, exist := query["schema"]; exist { schemaOnly = true @@ -1087,11 +1158,11 @@ func (api *APIServer) httpUploadHandler(w http.ResponseWriter, r *http.Request) return } - commandId, ctx := status.Current.Start(fullCommand) go func() { + commandId, _ := status.Current.Start(fullCommand) err, _ := api.metrics.ExecuteWithMetrics("upload", 0, func() error { b := backup.NewBackuper(cfg) - return b.Upload(name, diffFrom, diffFromRemote, tablePattern, partitionsToBackup, schemaOnly, resume, commandId) + return b.Upload(name, deleteSource, diffFrom, diffFromRemote, tablePattern, partitionsToBackup, schemaOnly, resume, api.cliApp.Version, commandId) }) if err != nil { log.Error().Msgf("Upload error: %v", err) @@ -1099,12 +1170,11 @@ func (api *APIServer) httpUploadHandler(w http.ResponseWriter, r *http.Request) api.errorCallback(context.Background(), err, callback) return } - if err := api.UpdateBackupMetrics(ctx, false); err != nil { - log.Error().Msgf("UpdateBackupMetrics return error: %v", err) - status.Current.Stop(commandId, err) - api.errorCallback(context.Background(), err, callback) - return - } + go func() { + if metricsErr := api.UpdateBackupMetrics(context.Background(), false); metricsErr != nil { + log.Error().Msgf("UpdateBackupMetrics return error: %v", metricsErr) + } + }() status.Current.Stop(commandId, nil) api.successCallback(context.Background(), callback) }() @@ -1124,6 +1194,7 @@ func (api *APIServer) httpUploadHandler(w http.ResponseWriter, r *http.Request) } var databaseMappingRE = regexp.MustCompile(`[\w+]:[\w+]`) +var tableMappingRE = regexp.MustCompile(`[\w+]:[\w+]`) // httpRestoreHandler - restore a backup from local storage func (api *APIServer) httpRestoreHandler(w http.ResponseWriter, r *http.Request) { @@ -1139,10 +1210,11 @@ func (api *APIServer) httpRestoreHandler(w http.ResponseWriter, r *http.Request) vars := mux.Vars(r) tablePattern := "" databaseMappingToRestore := make([]string, 0) + tableMappingToRestore := make([]string, 0) partitionsToBackup := make([]string, 0) schemaOnly := false dataOnly := false - dropTable := false + dropExists := false ignoreDependencies := false restoreRBAC := false restoreConfigs := false @@ -1168,9 +1240,27 @@ func (api *APIServer) httpRestoreHandler(w http.ResponseWriter, r *http.Request) fullCommand = fmt.Sprintf("%s --restore-database-mapping=\"%s\"", fullCommand, strings.Join(databaseMappingToRestore, ",")) } + + // https://github.com/Altinity/clickhouse-backup/issues/937 + if tableMappingQuery, exist := query["restore_table_mapping"]; exist { + for _, tableMapping := range tableMappingQuery { + mappingItems := strings.Split(tableMapping, ",") + for _, m := range mappingItems { + if strings.Count(m, ":") != 1 || !tableMappingRE.MatchString(m) { + api.writeError(w, http.StatusInternalServerError, "restore", fmt.Errorf("invalid values in restore_table_mapping %s", m)) + return + + } + } + tableMappingToRestore = append(tableMappingToRestore, mappingItems...) + } + + fullCommand = fmt.Sprintf("%s --restore-table-mapping=\"%s\"", fullCommand, strings.Join(tableMappingToRestore, ",")) + } + if partitions, exist := query["partitions"]; exist { - partitionsToBackup = partitions - fullCommand = fmt.Sprintf("%s --partitions=\"%s\"", fullCommand, strings.Join(partitions, ",")) + partitionsToBackup = append(partitionsToBackup, partitions...) + fullCommand = fmt.Sprintf("%s --partitions=\"%s\"", fullCommand, strings.Join(partitions, "\" --partitions=\"")) } if _, exist := query["schema"]; exist { schemaOnly = true @@ -1181,11 +1271,11 @@ func (api *APIServer) httpRestoreHandler(w http.ResponseWriter, r *http.Request) fullCommand += " --data" } if _, exist := query["drop"]; exist { - dropTable = true + dropExists = true fullCommand += " --drop" } if _, exist := query["rm"]; exist { - dropTable = true + dropExists = true fullCommand += " --rm" } if _, exists := query["ignore_dependencies"]; exists { @@ -1215,7 +1305,7 @@ func (api *APIServer) httpRestoreHandler(w http.ResponseWriter, r *http.Request) go func() { err, _ := api.metrics.ExecuteWithMetrics("restore", 0, func() error { b := backup.NewBackuper(api.config) - return b.Restore(name, tablePattern, databaseMappingToRestore, partitionsToBackup, schemaOnly, dataOnly, dropTable, ignoreDependencies, restoreRBAC, false, restoreConfigs, false, commandId) + return b.Restore(name, tablePattern, databaseMappingToRestore, tableMappingToRestore, partitionsToBackup, schemaOnly, dataOnly, dropExists, ignoreDependencies, restoreRBAC, false, restoreConfigs, false, api.cliApp.Version, commandId) }) status.Current.Stop(commandId, err) if err != nil { @@ -1262,8 +1352,8 @@ func (api *APIServer) httpDownloadHandler(w http.ResponseWriter, r *http.Request fullCommand = fmt.Sprintf("%s --tables=\"%s\"", fullCommand, tablePattern) } if partitions, exist := query["partitions"]; exist { - partitionsToBackup = partitions - fullCommand = fmt.Sprintf("%s --partitions=\"%s\"", fullCommand, strings.Join(partitions, ",")) + partitionsToBackup = append(partitionsToBackup, partitions...) + fullCommand = fmt.Sprintf("%s --partitions=\"%s\"", fullCommand, strings.Join(partitions, "\" --partitions=\"")) } if _, exist := query["schema"]; exist { schemaOnly = true @@ -1282,11 +1372,11 @@ func (api *APIServer) httpDownloadHandler(w http.ResponseWriter, r *http.Request return } - commandId, ctx := status.Current.Start(fullCommand) go func() { + commandId, _ := status.Current.Start(fullCommand) err, _ := api.metrics.ExecuteWithMetrics("download", 0, func() error { b := backup.NewBackuper(cfg) - return b.Download(name, tablePattern, partitionsToBackup, schemaOnly, resume, commandId) + return b.Download(name, tablePattern, partitionsToBackup, schemaOnly, resume, api.cliApp.Version, commandId) }) if err != nil { log.Error().Msgf("API /backup/download error: %v", err) @@ -1294,12 +1384,11 @@ func (api *APIServer) httpDownloadHandler(w http.ResponseWriter, r *http.Request api.errorCallback(context.Background(), err, callback) return } - if err := api.UpdateBackupMetrics(ctx, true); err != nil { - log.Error().Msgf("UpdateBackupMetrics return error: %v", err) - status.Current.Stop(commandId, err) - api.errorCallback(context.Background(), err, callback) - return - } + go func() { + if metricsErr := api.UpdateBackupMetrics(context.Background(), true); metricsErr != nil { + log.Error().Msgf("UpdateBackupMetrics return error: %v", metricsErr) + } + }() status.Current.Stop(commandId, nil) api.successCallback(context.Background(), callback) }() @@ -1344,8 +1433,8 @@ func (api *APIServer) httpDeleteHandler(w http.ResponseWriter, r *http.Request) return } go func() { - if err := api.UpdateBackupMetrics(context.Background(), vars["where"] == "local"); err != nil { - log.Error().Msgf("UpdateBackupMetrics return error: %v", err) + if metricsErr := api.UpdateBackupMetrics(context.Background(), vars["where"] == "local"); metricsErr != nil { + log.Error().Msgf("UpdateBackupMetrics return error: %v", metricsErr) } }() api.sendJSONEachRow(w, http.StatusOK, struct { @@ -1412,7 +1501,7 @@ func (api *APIServer) UpdateBackupMetrics(ctx context.Context, onlyLocal bool) e } } lastBackup := remoteBackups[numberBackupsRemote-1] - lastSizeRemote = lastBackup.DataSize + lastBackup.MetadataSize + lastBackup.ConfigSize + lastBackup.RBACSize + lastSizeRemote = lastBackup.GetFullSize() lastBackupCreateRemote = &lastBackup.CreationDate lastBackupUpload = &lastBackup.UploadDate api.metrics.LastBackupSizeRemote.Set(float64(lastSizeRemote)) @@ -1486,7 +1575,10 @@ func (api *APIServer) CreateIntegrationTables() error { return fmt.Errorf("can't connect to clickhouse: %w", err) } defer ch.Close() - port := strings.Split(api.config.API.ListenAddr, ":")[1] + port := "80" + if strings.Contains(api.config.API.ListenAddr, ":") { + port = api.config.API.ListenAddr[strings.Index(api.config.API.ListenAddr, ":")+1:] + } auth := "" if api.config.API.Username != "" || api.config.API.Password != "" { params := url.Values{} @@ -1510,12 +1602,24 @@ func (api *APIServer) CreateIntegrationTables() error { if version >= 21001000 { settings = "SETTINGS input_format_skip_unknown_fields=1" } + disks, err := ch.GetDisks(context.Background(), true) + if err != nil { + return err + } + defaultDataPath, err := ch.GetDefaultPath(disks) + if err != nil { + return err + } query := fmt.Sprintf("CREATE TABLE system.backup_actions (command String, start DateTime, finish DateTime, status String, error String) ENGINE=URL('%s://%s:%s/backup/actions%s', JSONEachRow) %s", schema, host, port, auth, settings) - if err := ch.CreateTable(clickhouse.Table{Database: "system", Name: "backup_actions"}, query, true, false, "", 0); err != nil { + if err := ch.CreateTable(clickhouse.Table{Database: "system", Name: "backup_actions"}, query, true, false, "", 0, defaultDataPath); err != nil { return err } query = fmt.Sprintf("CREATE TABLE system.backup_list (name String, created DateTime, size Int64, location String, required String, desc String) ENGINE=URL('%s://%s:%s/backup/list%s', JSONEachRow) %s", schema, host, port, auth, settings) - if err := ch.CreateTable(clickhouse.Table{Database: "system", Name: "backup_list"}, query, true, false, "", 0); err != nil { + if err := ch.CreateTable(clickhouse.Table{Database: "system", Name: "backup_list"}, query, true, false, "", 0, defaultDataPath); err != nil { + return err + } + query = fmt.Sprintf("CREATE TABLE system.backup_version (version String) ENGINE=URL('%s://%s:%s/backup/version%s', JSONEachRow) %s", schema, host, port, auth, settings) + if err := ch.CreateTable(clickhouse.Table{Database: "system", Name: "backup_version"}, query, true, false, "", 0, defaultDataPath); err != nil { return err } return nil @@ -1598,9 +1702,9 @@ func (api *APIServer) ResumeOperationsAfterRestart() error { if partitions, ok := params["partitions"]; ok && len(partitions.([]interface{})) > 0 { partitionsStr := make([]string, len(partitions.([]interface{}))) for j, v := range partitions.([]interface{}) { - partitionsStr[j] = v.(string) + partitionsStr[j] = fmt.Sprintf("--partitions=\"%s\"", v.(string)) } - args = append(args, fmt.Sprintf("--partitions=\"%s\"", strings.Join(partitionsStr, ","))) + args = append(args, partitionsStr...) } args = append(args, "--resumable=1", backupName) fullCommand := strings.Join(args, " ") @@ -1613,8 +1717,12 @@ func (api *APIServer) ResumeOperationsAfterRestart() error { if err != nil { return err } + if err = os.Remove(stateFile); err != nil { - return err + if api.config.General.BackupsToKeepLocal >= 0 { + return err + } + log.Warn().Str("operation", "ResumeOperationsAfterRestart").Msgf("remove %s return error: ", err) } default: return fmt.Errorf("unkown command for state file %s", stateFile) diff --git a/pkg/status/status.go b/pkg/status/status.go index f4c0a342..d7b3f186 100644 --- a/pkg/status/status.go +++ b/pkg/status/status.go @@ -8,7 +8,7 @@ import ( "sync" "time" - "github.com/Altinity/clickhouse-backup/pkg/common" + "github.com/Altinity/clickhouse-backup/v2/pkg/common" ) const ( @@ -70,16 +70,19 @@ func (status *AsyncStatus) CheckCommandInProgress(command string) bool { return false } +// InProgress any .Status == InProgressStatus command shall return true, https://github.com/Altinity/clickhouse-backup/issues/827 func (status *AsyncStatus) InProgress() bool { status.RLock() defer status.RUnlock() - n := len(status.commands) - 1 - if n < 0 { - log.Debug().Msgf("api.status.inProgress -> len(status.commands)=%d, inProgress=false", len(status.commands)) - return false + for n := range status.commands { + if status.commands[n].Status == InProgressStatus { + log.Debug().Msgf("api.status.inProgress -> status.commands[%d].Status == %s, inProgress=%v", n, status.commands[n].Status, status.commands[n].Status == InProgressStatus) + return true + } } - log.Debug().Msgf("api.status.inProgress -> status.commands[n].Status == %s, inProgress=%v", status.commands[n].Status, status.commands[n].Status == InProgressStatus) - return status.commands[n].Status == InProgressStatus + + log.Debug().Msgf("api.status.inProgress -> len(status.commands)=%d, inProgress=false", len(status.commands)) + return false } func (status *AsyncStatus) GetContextWithCancel(commandId int) (context.Context, context.CancelFunc, error) { diff --git a/pkg/storage/azblob.go b/pkg/storage/azblob.go index 44eca049..e15010ac 100644 --- a/pkg/storage/azblob.go +++ b/pkg/storage/azblob.go @@ -2,24 +2,24 @@ package storage import ( "context" - "crypto/rand" "crypto/sha256" "encoding/base64" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" "io" "net/url" "path" "strings" "time" - x "github.com/Altinity/clickhouse-backup/pkg/storage/azblob" + x "github.com/Altinity/clickhouse-backup/v2/pkg/storage/azblob" "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-blob-go/azblob" "github.com/Azure/go-autorest/autorest/adal" "github.com/Azure/go-autorest/autorest/azure" "github.com/pkg/errors" + "github.com/rs/zerolog/log" ) // AzureBlob - presents methods for manipulate data on Azure @@ -30,6 +30,13 @@ type AzureBlob struct { Config *config.AzureBlobConfig } +func (a *AzureBlob) logf(msg string, args ...interface{}) { + if a.Config.Debug { + log.Info().Msgf(msg, args...) + } else { + log.Debug().Msgf(msg, args...) + } +} func (a *AzureBlob) Kind() string { return "azblob" } @@ -119,17 +126,19 @@ func (a *AzureBlob) Connect(ctx context.Context) error { if err != nil && !isContainerAlreadyExists(err) { return err } - testName := make([]byte, 16) - if _, err := rand.Read(testName); err != nil { - return errors.Wrapf(err, "azblob: failed to generate test blob name") - } - testBlob := a.Container.NewBlockBlobURL(base64.URLEncoding.EncodeToString(testName)) - if _, err = testBlob.GetProperties(ctx, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}); err != nil { - if se, ok := err.(azblob.StorageError); !ok || se.ServiceCode() != azblob.ServiceCodeBlobNotFound { - return errors.Wrapf(err, "azblob: failed to access container %s", a.Config.Container) - } - } - + //testName := make([]byte, 16) + //if _, err := rand.Read(testName); err != nil { + // return errors.Wrapf(err, "azblob: failed to generate test blob name") + //} + //testNameStr := base64.URLEncoding.EncodeToString(testName) + //a.logf("AZBLOB->try to GetProbperties test blob: %s", testNameStr) + //testBlob := a.Container.NewBlockBlobURL(testNameStr) + //if _, err = testBlob.GetProperties(ctx, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}); err != nil { + // var se azblob.StorageError + // if !errors.As(err, &se) || se.ServiceCode() != azblob.ServiceCodeBlobNotFound { + // return errors.Wrapf(err, "azblob: failed to access container %s", a.Config.Container) + // } + //} if a.Config.SSEKey != "" { key, err := base64.StdEncoding.DecodeString(a.Config.SSEKey) if err != nil { @@ -143,8 +152,8 @@ func (a *AzureBlob) Connect(ctx context.Context) error { b64sha := base64.StdEncoding.EncodeToString(shakey[:]) a.CPK = azblob.NewClientProvidedKeyOptions(&b64key, &b64sha, nil) } - return nil } + return nil } func (a *AzureBlob) Close(ctx context.Context) error { @@ -152,7 +161,12 @@ func (a *AzureBlob) Close(ctx context.Context) error { } func (a *AzureBlob) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error) { - blob := a.Container.NewBlockBlobURL(path.Join(a.Config.Path, key)) + return a.GetFileReaderAbsolute(ctx, path.Join(a.Config.Path, key)) +} + +func (a *AzureBlob) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error) { + a.logf("AZBLOB->GetFileReaderAbsolute %s", key) + blob := a.Container.NewBlockBlobURL(key) r, err := blob.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, a.CPK) if err != nil { return nil, err @@ -165,7 +179,12 @@ func (a *AzureBlob) GetFileReaderWithLocalPath(ctx context.Context, key, _ strin } func (a *AzureBlob) PutFile(ctx context.Context, key string, r io.ReadCloser) error { - blob := a.Container.NewBlockBlobURL(path.Join(a.Config.Path, key)) + return a.PutFileAbsolute(ctx, path.Join(a.Config.Path, key), r) +} + +func (a *AzureBlob) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error { + a.logf("AZBLOB->PutFileAbsolute %s", key) + blob := a.Container.NewBlockBlobURL(key) bufferSize := a.Config.BufferSize // Configure the size of the rotating buffers that are used when uploading maxBuffers := a.Config.MaxBuffers // Configure the number of rotating buffers that are used when uploading _, err := x.UploadStreamToBlockBlob(ctx, r, blob, azblob.UploadStreamToBlockBlobOptions{BufferSize: bufferSize, MaxBuffers: maxBuffers}, a.CPK) @@ -173,22 +192,26 @@ func (a *AzureBlob) PutFile(ctx context.Context, key string, r io.ReadCloser) er } func (a *AzureBlob) DeleteFile(ctx context.Context, key string) error { + a.logf("AZBLOB->DeleteFile %s", key) blob := a.Container.NewBlockBlobURL(path.Join(a.Config.Path, key)) _, err := blob.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}) return err } func (a *AzureBlob) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error { + a.logf("AZBLOB->DeleteFileFromObjectDiskBackup %s", key) blob := a.Container.NewBlockBlobURL(path.Join(a.Config.ObjectDiskPath, key)) _, err := blob.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}) return err } func (a *AzureBlob) StatFile(ctx context.Context, key string) (RemoteFile, error) { + a.logf("AZBLOB->StatFile %s", key) blob := a.Container.NewBlockBlobURL(path.Join(a.Config.Path, key)) r, err := blob.GetProperties(ctx, azblob.BlobAccessConditions{}, a.CPK) if err != nil { - if se, ok := err.(azblob.StorageError); !ok || se.ServiceCode() != azblob.ServiceCodeBlobNotFound { + var se azblob.StorageError + if !errors.As(err, &se) || se.ServiceCode() != azblob.ServiceCodeBlobNotFound { return nil, err } return nil, ErrNotFound @@ -202,6 +225,11 @@ func (a *AzureBlob) StatFile(ctx context.Context, key string) (RemoteFile, error func (a *AzureBlob) Walk(ctx context.Context, azPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error { prefix := path.Join(a.Config.Path, azPath) + return a.WalkAbsolute(ctx, prefix, recursive, process) +} + +func (a *AzureBlob) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error { + a.logf("AZBLOB->WalkAbsolute %s", prefix) if prefix == "" || prefix == "/" { prefix = "" } else { @@ -270,8 +298,9 @@ func (a *AzureBlob) Walk(ctx context.Context, azPath string, recursive bool, pro return nil } -func (a *AzureBlob) CopyObject(ctx context.Context, srcBucket, srcKey, dstKey string) (int64, error) { +func (a *AzureBlob) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error) { dstKey = path.Join(a.Config.ObjectDiskPath, dstKey) + a.logf("AZBLOB->CopyObject %s/%s -> %s/%s", srcBucket, srcKey, a.Config.Container, dstKey) srcURLString := fmt.Sprintf("%s://%s.%s/%s/%s", a.Config.EndpointSchema, a.Config.AccountName, a.Config.EndpointSuffix, srcBucket, srcKey) srcURL, err := url.Parse(srcURLString) if err != nil { @@ -300,7 +329,9 @@ func (a *AzureBlob) CopyObject(ctx context.Context, srcBucket, srcKey, dstKey st copyStatus = dstMeta.CopyStatus() copyStatusDesc = dstMeta.CopyStatusDescription() size = dstMeta.ContentLength() - pollCount++ + if pollCount < 8 { + pollCount++ + } } if copyStatus == azblob.CopyStatusFailed { return 0, fmt.Errorf("azblob->CopyObject got CopyStatusFailed %s", copyStatusDesc) @@ -328,7 +359,8 @@ func (f *azureBlobFile) LastModified() time.Time { func isContainerAlreadyExists(err error) bool { if err != nil { - if storageErr, ok := err.(azblob.StorageError); ok { // This error is a Service-specific + var storageErr azblob.StorageError + if errors.As(err, &storageErr) { // This error is a Service-specific switch storageErr.ServiceCode() { // Compare serviceCode to ServiceCodeXxx constants case azblob.ServiceCodeContainerAlreadyExists: return true diff --git a/pkg/storage/azblob/chunkwriting.go b/pkg/storage/azblob/chunkwriting.go index efbb8ce6..460b093a 100644 --- a/pkg/storage/azblob/chunkwriting.go +++ b/pkg/storage/azblob/chunkwriting.go @@ -145,11 +145,11 @@ func (c *copier) sendChunk() error { id: c.id.next(), } return nil - case err != nil && (err == io.EOF || err == io.ErrUnexpectedEOF) && n == 0: + case err != nil && (err == io.EOF || errors.Is(err, io.ErrUnexpectedEOF)) && n == 0: return io.EOF } - if err == io.EOF || err == io.ErrUnexpectedEOF { + if err == io.EOF || errors.Is(err, io.ErrUnexpectedEOF) { c.ch <- copierChunk{ buffer: buffer[0:n], id: c.id.next(), diff --git a/pkg/storage/cos.go b/pkg/storage/cos.go index 31aa9944..d0a953c7 100644 --- a/pkg/storage/cos.go +++ b/pkg/storage/cos.go @@ -2,8 +2,9 @@ package storage import ( "context" + "errors" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" "io" "net/http" "net/url" @@ -62,7 +63,8 @@ func (c *COS) StatFile(ctx context.Context, key string) (RemoteFile, error) { // file max size is 5Gb resp, err := c.client.Object.Get(ctx, path.Join(c.Config.Path, key), nil) if err != nil { - cosErr, ok := err.(*cos.ErrorResponse) + var cosErr *cos.ErrorResponse + ok := errors.As(err, &cosErr) if ok && cosErr.Code == "NoSuchKey" { return nil, ErrNotFound } @@ -84,6 +86,10 @@ func (c *COS) DeleteFile(ctx context.Context, key string) error { func (c *COS) Walk(ctx context.Context, cosPath string, recursive bool, process func(context.Context, RemoteFile) error) error { // COS needs prefix ended with "/". prefix := path.Join(c.Config.Path, cosPath) + "/" + return c.WalkAbsolute(ctx, prefix, recursive, process) +} + +func (c *COS) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error { delimiter := "" if !recursive { @@ -130,7 +136,11 @@ func (c *COS) Walk(ctx context.Context, cosPath string, recursive bool, process } func (c *COS) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error) { - resp, err := c.client.Object.Get(ctx, path.Join(c.Config.Path, key), nil) + return c.GetFileReaderAbsolute(ctx, path.Join(c.Config.Path, key)) +} + +func (c *COS) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error) { + resp, err := c.client.Object.Get(ctx, key, nil) if err != nil { return nil, err } @@ -142,11 +152,15 @@ func (c *COS) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io } func (c *COS) PutFile(ctx context.Context, key string, r io.ReadCloser) error { - _, err := c.client.Object.Put(ctx, path.Join(c.Config.Path, key), r, nil) + return c.PutFileAbsolute(ctx, path.Join(c.Config.Path, key), r) +} + +func (c *COS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error { + _, err := c.client.Object.Put(ctx, key, r, nil) return err } -func (c *COS) CopyObject(ctx context.Context, srcBucket, srcKey, dstKey string) (int64, error) { +func (c *COS) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error) { return 0, fmt.Errorf("CopyObject not imlemented for %s", c.Kind()) } diff --git a/pkg/storage/ftp.go b/pkg/storage/ftp.go index 047e937c..c3b4adcf 100644 --- a/pkg/storage/ftp.go +++ b/pkg/storage/ftp.go @@ -11,7 +11,7 @@ import ( "sync" "time" - "github.com/Altinity/clickhouse-backup/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" "github.com/jlaffaye/ftp" "github.com/jolestar/go-commons-pool/v2" "github.com/rs/zerolog/log" @@ -42,12 +42,12 @@ func (f *FTP) Connect(ctx context.Context) error { options = append(options, ftp.DialWithDebugOutput(os.Stdout)) } if f.Config.TLS { - tlsConfig := tls.Config{} + tlsConfig := tls.Config{InsecureSkipVerify: f.Config.SkipTLSVerify} options = append(options, ftp.DialWithTLS(&tlsConfig)) } f.clients = pool.NewObjectPoolWithDefaultConfig(ctx, &ftpPoolFactory{options: options, ftp: f}) if f.Config.Concurrency > 1 { - f.clients.Config.MaxTotal = int(f.Config.Concurrency) * 3 + f.clients.Config.MaxTotal = int(f.Config.Concurrency) * 4 } f.dirCacheMutex.Lock() @@ -123,11 +123,15 @@ func (f *FTP) DeleteFile(ctx context.Context, key string) error { } func (f *FTP) Walk(ctx context.Context, ftpPath string, recursive bool, process func(context.Context, RemoteFile) error) error { + prefix := path.Join(f.Config.Path, ftpPath) + return f.WalkAbsolute(ctx, prefix, recursive, process) +} + +func (f *FTP) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error { client, err := f.getConnectionFromPool(ctx, "Walk") if err != nil { return err } - prefix := path.Join(f.Config.Path, ftpPath) if !recursive { entries, err := client.List(prefix) f.returnConnectionToPool(ctx, "Walk", client) @@ -174,12 +178,15 @@ func (f *FTP) Walk(ctx context.Context, ftpPath string, recursive bool, process } func (f *FTP) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error) { - log.Debug().Msgf("GetFileReader key=%s", key) + return f.GetFileReaderAbsolute(ctx, path.Join(f.Config.Path, key)) +} +func (f *FTP) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error) { + log.Debug().Msgf("GetFileReaderAbsolute key=%s", key) client, err := f.getConnectionFromPool(ctx, "GetFileReader") if err != nil { return nil, err } - resp, err := client.Retr(path.Join(f.Config.Path, key)) + resp, err := client.Retr(key) return &FTPFileReader{ Response: resp, pool: f, @@ -193,21 +200,24 @@ func (f *FTP) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io } func (f *FTP) PutFile(ctx context.Context, key string, r io.ReadCloser) error { - log.Debug().Msgf("PutFile key=%s", key) + return f.PutFileAbsolute(ctx, path.Join(f.Config.Path, key), r) +} + +func (f *FTP) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error { + log.Debug().Msgf("PutFileAbsolute key=%s", key) client, err := f.getConnectionFromPool(ctx, "PutFile") defer f.returnConnectionToPool(ctx, "PutFile", client) if err != nil { return err } - k := path.Join(f.Config.Path, key) - err = f.MkdirAll(path.Dir(k), client) + err = f.MkdirAll(path.Dir(key), client) if err != nil { return err } - return client.Stor(k, r) + return client.Stor(key, r) } -func (f *FTP) CopyObject(ctx context.Context, srcBucket, srcKey, dstKey string) (int64, error) { +func (f *FTP) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error) { return 0, fmt.Errorf("CopyObject not imlemented for %s", f.Kind()) } diff --git a/pkg/storage/gcs.go b/pkg/storage/gcs.go index 6878e57b..6553daec 100644 --- a/pkg/storage/gcs.go +++ b/pkg/storage/gcs.go @@ -2,35 +2,44 @@ package storage import ( "context" + "crypto/tls" "encoding/base64" "errors" "fmt" "io" + "net" "net/http" "path" "strings" "time" - "github.com/Altinity/clickhouse-backup/pkg/config" + "google.golang.org/api/iterator" + + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + pool "github.com/jolestar/go-commons-pool/v2" "google.golang.org/api/option/internaloption" "cloud.google.com/go/storage" "github.com/rs/zerolog/log" - "google.golang.org/api/iterator" "google.golang.org/api/option" googleHTTPTransport "google.golang.org/api/transport/http" ) // GCS - presents methods for manipulate data on GCS type GCS struct { - client *storage.Client - Config *config.GCSConfig + client *storage.Client + Config *config.GCSConfig + clientPool *pool.ObjectPool } type debugGCSTransport struct { base http.RoundTripper } +type clientObject struct { + Client *storage.Client +} + func (w debugGCSTransport) RoundTrip(r *http.Request) (*http.Response, error) { logMsg := fmt.Sprintf(">>> [GCS_REQUEST] >>> %v %v\n", r.Method, r.URL.String()) for h, values := range r.Header { @@ -59,6 +68,19 @@ func (gcs *GCS) Kind() string { return "GCS" } +type rewriteTransport struct { + base http.RoundTripper +} + +// forces requests to target varnish and use HTTP, required to get uploading +// via varnish working +func (r rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL.Scheme == "https" { + req.URL.Scheme = "http" + } + return r.base.RoundTrip(req) +} + // Connect - connect to GCS func (gcs *GCS) Connect(ctx context.Context) error { var err error @@ -68,15 +90,58 @@ func (gcs *GCS) Connect(ctx context.Context) error { if gcs.Config.Endpoint != "" { endpoint = gcs.Config.Endpoint - clientOptions = append([]option.ClientOption{option.WithoutAuthentication()}, clientOptions...) clientOptions = append(clientOptions, option.WithEndpoint(endpoint)) - } else if gcs.Config.CredentialsJSON != "" { + } + + if gcs.Config.CredentialsJSON != "" { clientOptions = append(clientOptions, option.WithCredentialsJSON([]byte(gcs.Config.CredentialsJSON))) } else if gcs.Config.CredentialsJSONEncoded != "" { d, _ := base64.StdEncoding.DecodeString(gcs.Config.CredentialsJSONEncoded) clientOptions = append(clientOptions, option.WithCredentialsJSON(d)) } else if gcs.Config.CredentialsFile != "" { clientOptions = append(clientOptions, option.WithCredentialsFile(gcs.Config.CredentialsFile)) + } else if gcs.Config.SkipCredentials { + clientOptions = append(clientOptions, option.WithoutAuthentication()) + } + + if gcs.Config.ForceHttp { + customTransport := &http.Transport{ + WriteBufferSize: 128 * 1024, + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 1, + MaxIdleConnsPerHost: 1, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + // must set ForceAttemptHTTP2 to false so that when a custom TLSClientConfig + // is provided Golang does not setup HTTP/2 transport + customTransport.ForceAttemptHTTP2 = false + customTransport.TLSClientConfig = &tls.Config{ + NextProtos: []string{"http/1.1"}, + } + // These clientOptions are passed in by storage.NewClient. However, to set a custom HTTP client + // we must pass all these in manually. + + if gcs.Config.Endpoint == "" { + clientOptions = append([]option.ClientOption{option.WithScopes(storage.ScopeFullControl)}, clientOptions...) + } + clientOptions = append(clientOptions, internaloption.WithDefaultEndpoint(endpoint)) + + customRoundTripper := &rewriteTransport{base: customTransport} + gcpTransport, _, err := googleHTTPTransport.NewClient(ctx, clientOptions...) + transport, err := googleHTTPTransport.NewTransport(ctx, customRoundTripper, clientOptions...) + gcpTransport.Transport = transport + if err != nil { + return fmt.Errorf("failed to create GCP transport: %v", err) + } + + clientOptions = append(clientOptions, option.WithHTTPClient(gcpTransport)) + } if gcs.Config.Debug { @@ -96,16 +161,31 @@ func (gcs *GCS) Connect(ctx context.Context) error { clientOptions = append(clientOptions, option.WithHTTPClient(debugClient)) } + factory := pool.NewPooledObjectFactorySimple( + func(context.Context) (interface{}, error) { + sClient, err := storage.NewClient(ctx, clientOptions...) + if err != nil { + return nil, err + } + return &clientObject{Client: sClient}, nil + }) + gcs.clientPool = pool.NewObjectPoolWithDefaultConfig(ctx, factory) + gcs.clientPool.Config.MaxTotal = gcs.Config.ClientPoolSize * 3 gcs.client, err = storage.NewClient(ctx, clientOptions...) return err } func (gcs *GCS) Close(ctx context.Context) error { + gcs.clientPool.Close(ctx) return gcs.client.Close() } func (gcs *GCS) Walk(ctx context.Context, gcsPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error { rootPath := path.Join(gcs.Config.Path, gcsPath) + return gcs.WalkAbsolute(ctx, rootPath, recursive, process) +} + +func (gcs *GCS) WalkAbsolute(ctx context.Context, rootPath string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error { prefix := rootPath + "/" if rootPath == "/" { prefix = "" @@ -120,36 +200,53 @@ func (gcs *GCS) Walk(ctx context.Context, gcsPath string, recursive bool, proces }) for { object, err := it.Next() - if errors.Is(err, iterator.Done) { - return nil - } - if err != nil { - return err - } - if object.Prefix != "" { + switch { + case err == nil: + if object.Prefix != "" { + if err := process(ctx, &gcsFile{ + name: strings.TrimPrefix(object.Prefix, rootPath), + }); err != nil { + return err + } + continue + } if err := process(ctx, &gcsFile{ - name: strings.TrimPrefix(object.Prefix, rootPath), + size: object.Size, + lastModified: object.Updated, + name: strings.TrimPrefix(object.Name, rootPath), }); err != nil { return err } - continue - } - if err := process(ctx, &gcsFile{ - size: object.Size, - lastModified: object.Updated, - name: strings.TrimPrefix(object.Name, rootPath), - }); err != nil { + case errors.Is(err, iterator.Done): + return nil + default: return err } } } func (gcs *GCS) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error) { - obj := gcs.client.Bucket(gcs.Config.Bucket).Object(path.Join(gcs.Config.Path, key)) + return gcs.GetFileReaderAbsolute(ctx, path.Join(gcs.Config.Path, key)) +} + +func (gcs *GCS) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error) { + pClientObj, err := gcs.clientPool.BorrowObject(ctx) + if err != nil { + log.Error().Msgf("gcs.GetFileReader: gcs.clientPool.BorrowObject error: %+v", err) + return nil, err + } + pClient := pClientObj.(*clientObject).Client + obj := pClient.Bucket(gcs.Config.Bucket).Object(key) reader, err := obj.NewReader(ctx) if err != nil { + if pErr := gcs.clientPool.InvalidateObject(ctx, pClientObj); pErr != nil { + log.Warn().Msgf("gcs.GetFileReader: gcs.clientPool.InvalidateObject error: %v ", pErr) + } return nil, err } + if pErr := gcs.clientPool.ReturnObject(ctx, pClientObj); pErr != nil { + log.Warn().Msgf("gcs.GetFileReader: gcs.clientPool.ReturnObject error: %v ", pErr) + } return reader, nil } @@ -158,21 +255,40 @@ func (gcs *GCS) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) ( } func (gcs *GCS) PutFile(ctx context.Context, key string, r io.ReadCloser) error { - key = path.Join(gcs.Config.Path, key) - obj := gcs.client.Bucket(gcs.Config.Bucket).Object(key) + return gcs.PutFileAbsolute(ctx, path.Join(gcs.Config.Path, key), r) +} + +func (gcs *GCS) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error { + pClientObj, err := gcs.clientPool.BorrowObject(ctx) + if err != nil { + log.Error().Msgf("gcs.PutFile: gcs.clientPool.BorrowObject error: %+v", err) + return err + } + pClient := pClientObj.(*clientObject).Client + obj := pClient.Bucket(gcs.Config.Bucket).Object(key) writer := obj.NewWriter(ctx) + writer.ChunkSize = gcs.Config.ChunkSize writer.StorageClass = gcs.Config.StorageClass + writer.ChunkRetryDeadline = 60 * time.Minute if len(gcs.Config.ObjectLabels) > 0 { writer.Metadata = gcs.Config.ObjectLabels } defer func() { - if err := writer.Close(); err != nil { - log.Warn().Msgf("can't close writer: %+v", err) + if err := gcs.clientPool.ReturnObject(ctx, pClientObj); err != nil { + log.Warn().Msgf("gcs.PutFile: gcs.clientPool.ReturnObject error: %+v", err) } }() - buffer := make([]byte, 512*1024) - _, err := io.CopyBuffer(writer, r, buffer) - return err + buffer := make([]byte, 128*1024) + _, err = io.CopyBuffer(writer, r, buffer) + if err != nil { + log.Warn().Msgf("gcs.PutFile: can't copy buffer: %+v", err) + return err + } + if err = writer.Close(); err != nil { + log.Warn().Msgf("gcs.PutFile: can't close writer: %+v", err) + return err + } + return nil } func (gcs *GCS) StatFile(ctx context.Context, key string) (RemoteFile, error) { @@ -191,8 +307,24 @@ func (gcs *GCS) StatFile(ctx context.Context, key string) (RemoteFile, error) { } func (gcs *GCS) deleteKey(ctx context.Context, key string) error { - object := gcs.client.Bucket(gcs.Config.Bucket).Object(key) - return object.Delete(ctx) + pClientObj, err := gcs.clientPool.BorrowObject(ctx) + if err != nil { + log.Error().Msgf("gcs.deleteKey: gcs.clientPool.BorrowObject error: %+v", err) + return err + } + pClient := pClientObj.(*clientObject).Client + object := pClient.Bucket(gcs.Config.Bucket).Object(key) + err = object.Delete(ctx) + if err != nil { + if pErr := gcs.clientPool.InvalidateObject(ctx, pClientObj); pErr != nil { + log.Warn().Msgf("gcs.deleteKey: gcs.clientPool.InvalidateObject error: %+v", pErr) + } + return err + } + if pErr := gcs.clientPool.ReturnObject(ctx, pClientObj); pErr != nil { + log.Warn().Msgf("gcs.deleteKey: gcs.clientPool.ReturnObject error: %+v", pErr) + } + return nil } func (gcs *GCS) DeleteFile(ctx context.Context, key string) error { @@ -205,18 +337,33 @@ func (gcs *GCS) DeleteFileFromObjectDiskBackup(ctx context.Context, key string) return gcs.deleteKey(ctx, key) } -func (gcs *GCS) CopyObject(ctx context.Context, srcBucket, srcKey, dstKey string) (int64, error) { +func (gcs *GCS) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error) { dstKey = path.Join(gcs.Config.ObjectDiskPath, dstKey) - src := gcs.client.Bucket(srcBucket).Object(srcKey) - dst := gcs.client.Bucket(gcs.Config.Bucket).Object(dstKey) + log.Debug().Msgf("GCS->CopyObject %s/%s -> %s/%s", srcBucket, srcKey, gcs.Config.Bucket, dstKey) + pClientObj, err := gcs.clientPool.BorrowObject(ctx) + if err != nil { + log.Error().Msgf("gcs.CopyObject: gcs.clientPool.BorrowObject error: %+v", err) + return 0, err + } + pClient := pClientObj.(*clientObject).Client + src := pClient.Bucket(srcBucket).Object(srcKey) + dst := pClient.Bucket(gcs.Config.Bucket).Object(dstKey) attrs, err := src.Attrs(ctx) if err != nil { + if pErr := gcs.clientPool.InvalidateObject(ctx, pClientObj); pErr != nil { + log.Warn().Msgf("gcs.CopyObject: gcs.clientPool.InvalidateObject error: %+v", pErr) + } return 0, err } if _, err = dst.CopierFrom(src).Run(ctx); err != nil { + if pErr := gcs.clientPool.InvalidateObject(ctx, pClientObj); pErr != nil { + log.Warn().Msgf("gcs.CopyObject: gcs.clientPool.InvalidateObject error: %+v", pErr) + } return 0, err } - log.Debug().Msgf("GCS->CopyObject %s/%s -> %s/%s", srcBucket, srcKey, gcs.Config.Bucket, dstKey) + if pErr := gcs.clientPool.ReturnObject(ctx, pClientObj); pErr != nil { + log.Warn().Msgf("gcs.CopyObject: gcs.clientPool.ReturnObject error: %+v", pErr) + } return attrs.Size, nil } diff --git a/pkg/storage/general.go b/pkg/storage/general.go index 9f46b4af..2d9a78af 100644 --- a/pkg/storage/general.go +++ b/pkg/storage/general.go @@ -14,11 +14,10 @@ import ( "sync" "time" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/metadata" - "github.com/Altinity/clickhouse-backup/pkg/progressbar" - "github.com/Altinity/clickhouse-backup/pkg/utils" + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" + "github.com/djherbis/buffer" "github.com/djherbis/nio/v3" "github.com/eapache/go-resiliency/retrier" @@ -29,7 +28,7 @@ import ( const ( // BufferSize - size of ring buffer between stream handlers - BufferSize = 512 * 1024 + BufferSize = 128 * 1024 ) type readerWrapperForContext func(p []byte) (n int, err error) @@ -40,62 +39,22 @@ func (readerWrapper readerWrapperForContext) Read(p []byte) (n int, err error) { type Backup struct { metadata.BackupMetadata - Legacy bool - FileExtension string - Broken string - UploadDate time.Time + Broken string + UploadDate time.Time `json:"upload_date"` } type BackupDestination struct { RemoteStorage - compressionFormat string - compressionLevel int - disableProgressBar bool + compressionFormat string + compressionLevel int } var metadataCacheLock sync.RWMutex -func (bd *BackupDestination) RemoveOldBackups(ctx context.Context, keep int) error { - if keep < 1 { - return nil - } - start := time.Now() - backupList, err := bd.BackupList(ctx, true, "") - if err != nil { - return err - } - backupsToDelete := GetBackupsToDelete(backupList, keep) - log.Info().Fields(map[string]interface{}{ - "operation": "RemoveOldBackups", - "duration": utils.HumanizeDuration(time.Since(start)), - }).Msg("calculate backup list for delete") - for _, backupToDelete := range backupsToDelete { - startDelete := time.Now() - if err := bd.RemoveBackup(ctx, backupToDelete); err != nil { - log.Warn().Msgf("can't delete %s return error : %v", backupToDelete.BackupName, err) - } - log.Info().Fields(map[string]interface{}{ - "operation": "RemoveOldBackups", - "location": "remote", - "backup": backupToDelete.BackupName, - "duration": utils.HumanizeDuration(time.Since(startDelete)), - }).Msg("done") - } - log.Info().Fields(map[string]interface{}{ - "operation": "RemoveOldBackups", - "duration": utils.HumanizeDuration(time.Since(start)), - }).Msg("done") - return nil -} - -func (bd *BackupDestination) RemoveBackup(ctx context.Context, backup Backup) error { +func (bd *BackupDestination) RemoveBackupRemote(ctx context.Context, backup Backup) error { if bd.Kind() == "SFTP" || bd.Kind() == "FTP" { return bd.DeleteFile(ctx, backup.BackupName) } - if backup.Legacy { - archiveName := fmt.Sprintf("%s.%s", backup.BackupName, backup.FileExtension) - return bd.DeleteFile(ctx, archiveName) - } return bd.Walk(ctx, backup.BackupName+"/", true, func(ctx context.Context, f RemoteFile) error { if bd.Kind() == "azblob" { if f.Size() > 0 || !f.LastModified().IsZero() { @@ -108,15 +67,6 @@ func (bd *BackupDestination) RemoveBackup(ctx context.Context, backup Backup) er }) } -func isLegacyBackup(backupName string) (bool, string, string) { - for _, suffix := range config.ArchiveExtensions { - if strings.HasSuffix(backupName, "."+suffix) { - return true, strings.TrimSuffix(backupName, "."+suffix), suffix - } - } - return false, backupName, "" -} - func (bd *BackupDestination) loadMetadataCache(ctx context.Context) (map[string]Backup, error) { listCacheFile := path.Join(os.TempDir(), fmt.Sprintf(".clickhouse-backup-metadata.cache.%s", bd.Kind())) listCache := map[string]Backup{} @@ -209,21 +159,8 @@ func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, if err != nil { return nil, err } + cacheMiss := false err = bd.Walk(ctx, "/", false, func(ctx context.Context, o RemoteFile) error { - // Legacy backup - if ok, backupName, fileExtension := isLegacyBackup(strings.TrimPrefix(o.Name(), "/")); ok { - result = append(result, Backup{ - metadata.BackupMetadata{ - BackupName: backupName, - DataSize: uint64(o.Size()), - }, - true, - fileExtension, - "", - o.LastModified(), - }) - return nil - } backupName := strings.Trim(o.Name(), "/") if !parseMetadata || (parseMetadataOnly != "" && parseMetadataOnly != backupName) { if cachedMetadata, isCached := listCache[backupName]; isCached { @@ -233,7 +170,6 @@ func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, BackupMetadata: metadata.BackupMetadata{ BackupName: backupName, }, - Legacy: false, }) } return nil @@ -248,8 +184,6 @@ func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, metadata.BackupMetadata{ BackupName: backupName, }, - false, - "", "broken (can't stat metadata.json)", o.LastModified(), // folder } @@ -262,8 +196,6 @@ func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, metadata.BackupMetadata{ BackupName: backupName, }, - false, - "", "broken (can't open metadata.json)", o.LastModified(), // folder } @@ -276,8 +208,6 @@ func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, metadata.BackupMetadata{ BackupName: backupName, }, - false, - "", "broken (can't read metadata.json)", o.LastModified(), // folder } @@ -293,18 +223,15 @@ func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, metadata.BackupMetadata{ BackupName: backupName, }, - false, - "", "broken (bad metadata.json)", o.LastModified(), // folder } result = append(result, brokenBackup) return nil } - goodBackup := Backup{ - m, false, "", "", mf.LastModified(), - } + goodBackup := Backup{m, "", mf.LastModified()} listCache[backupName] = goodBackup + cacheMiss = true result = append(result, goodBackup) return nil }) @@ -318,23 +245,24 @@ func (bd *BackupDestination) BackupList(ctx context.Context, parseMetadata bool, sort.SliceStable(result, func(i, j int) bool { return result[i].UploadDate.Before(result[j].UploadDate) }) - if err = bd.saveMetadataCache(ctx, listCache, result); err != nil { - return nil, err + if cacheMiss || len(result) < len(listCache) { + if err = bd.saveMetadataCache(ctx, listCache, result); err != nil { + return nil, fmt.Errorf("bd.saveMetadataCache return error: %v", err) + } } - return result, err + return result, nil } -func (bd *BackupDestination) DownloadCompressedStream(ctx context.Context, remotePath string, localPath string) error { +func (bd *BackupDestination) DownloadCompressedStream(ctx context.Context, remotePath string, localPath string, maxSpeed uint64) error { if err := os.MkdirAll(localPath, 0750); err != nil { return err } // get this first as GetFileReader blocks the ftp control channel - file, err := bd.StatFile(ctx, remotePath) + remoteFileInfo, err := bd.StatFile(ctx, remotePath) if err != nil { return err } - filesize := file.Size() - + startTime := time.Now() reader, err := bd.GetFileReaderWithLocalPath(ctx, remotePath, localPath) if err != nil { return err @@ -352,11 +280,8 @@ func (bd *BackupDestination) DownloadCompressedStream(ctx context.Context, remot } }() - bar := progressbar.StartNewByteBar(!bd.disableProgressBar, filesize) buf := buffer.New(BufferSize) - defer bar.Finish() bufReader := nio.NewReader(reader, buf) - proxyReader := bar.NewProxyReader(bufReader) compressionFormat := bd.compressionFormat if !checkArchiveExtension(path.Ext(remotePath), compressionFormat) { log.Warn().Msgf("remote file backup extension %s not equal with %s", remotePath, compressionFormat) @@ -366,7 +291,7 @@ func (bd *BackupDestination) DownloadCompressedStream(ctx context.Context, remot if err != nil { return err } - if err := z.Extract(ctx, proxyReader, nil, func(ctx context.Context, file archiver.File) error { + if err := z.Extract(ctx, bufReader, nil, func(ctx context.Context, file archiver.File) error { f, err := file.Open() if err != nil { return fmt.Errorf("can't open %s", file.NameInArchive) @@ -405,15 +330,11 @@ func (bd *BackupDestination) DownloadCompressedStream(ctx context.Context, remot }); err != nil { return err } + bd.throttleSpeed(startTime, remoteFileInfo.Size(), maxSpeed) return nil } -func (bd *BackupDestination) UploadCompressedStream(ctx context.Context, baseLocalPath string, files []string, remotePath string) error { - if _, err := bd.StatFile(ctx, remotePath); err != nil { - if err != ErrNotFound && !os.IsNotExist(err) { - return err - } - } +func (bd *BackupDestination) UploadCompressedStream(ctx context.Context, baseLocalPath string, files []string, remotePath string, maxSpeed uint64) error { var totalBytes int64 for _, filename := range files { fInfo, err := os.Stat(path.Join(baseLocalPath, filename)) @@ -424,12 +345,10 @@ func (bd *BackupDestination) UploadCompressedStream(ctx context.Context, baseLoc totalBytes += fInfo.Size() } } - bar := progressbar.StartNewByteBar(!bd.disableProgressBar, totalBytes) - defer bar.Finish() pipeBuffer := buffer.New(BufferSize) body, w := nio.Pipe(pipeBuffer) g, ctx := errgroup.WithContext(ctx) - + startTime := time.Now() var writerErr, readerErr error g.Go(func() error { defer func() { @@ -457,7 +376,7 @@ func (bd *BackupDestination) UploadCompressedStream(ctx context.Context, baseLoc if !info.Mode().IsRegular() { continue } - bar.Add64(info.Size()) + file := archiver.File{ FileInfo: info, NameInArchive: f, @@ -488,30 +407,21 @@ func (bd *BackupDestination) UploadCompressedStream(ctx context.Context, baseLoc readerErr = bd.PutFile(ctx, remotePath, body) return readerErr }) - return g.Wait() + if waitErr := g.Wait(); waitErr != nil { + return waitErr + } + bd.throttleSpeed(startTime, totalBytes, maxSpeed) + return nil } -func (bd *BackupDestination) DownloadPath(ctx context.Context, size int64, remotePath string, localPath string, RetriesOnFailure int, RetriesDuration time.Duration) error { - var bar *progressbar.Bar - if !bd.disableProgressBar { - totalBytes := size - if size == 0 { - if err := bd.Walk(ctx, remotePath, true, func(ctx context.Context, f RemoteFile) error { - totalBytes += f.Size() - return nil - }); err != nil { - return err - } - } - bar = progressbar.StartNewByteBar(!bd.disableProgressBar, totalBytes) - defer bar.Finish() - } +func (bd *BackupDestination) DownloadPath(ctx context.Context, remotePath string, localPath string, RetriesOnFailure int, RetriesDuration time.Duration, maxSpeed uint64) error { return bd.Walk(ctx, remotePath, true, func(ctx context.Context, f RemoteFile) error { if bd.Kind() == "SFTP" && (f.Name() == "." || f.Name() == "..") { return nil } retry := retrier.New(retrier.ConstantBackoff(RetriesOnFailure, RetriesDuration), nil) err := retry.RunCtx(ctx, func(ctx context.Context) error { + startTime := time.Now() r, err := bd.GetFileReader(ctx, path.Join(remotePath, f.Name())) if err != nil { log.Error().Err(err).Send() @@ -528,7 +438,7 @@ func (bd *BackupDestination) DownloadPath(ctx context.Context, size int64, remot log.Error().Err(err).Send() return err } - if _, err := io.CopyBuffer(dst, r, nil); err != nil { + if _, err := io.Copy(dst, r); err != nil { log.Error().Err(err).Send() return err } @@ -540,38 +450,33 @@ func (bd *BackupDestination) DownloadPath(ctx context.Context, size int64, remot log.Error().Err(err).Send() return err } + + if dstFileInfo, err := os.Stat(dstFilePath); err == nil { + bd.throttleSpeed(startTime, dstFileInfo.Size(), maxSpeed) + } else { + return err + } + return nil }) if err != nil { return err } - if !bd.disableProgressBar { - bar.Add64(f.Size()) - } return nil }) } -func (bd *BackupDestination) UploadPath(ctx context.Context, size int64, baseLocalPath string, files []string, remotePath string, RetriesOnFailure int, RetriesDuration time.Duration) (int64, error) { - var bar *progressbar.Bar - totalBytes := size - if size == 0 { - for _, filename := range files { - fInfo, err := os.Stat(filepath.Clean(path.Join(baseLocalPath, filename))) - if err != nil { - return 0, err - } - if fInfo.Mode().IsRegular() { - totalBytes += fInfo.Size() - } - } - } - if !bd.disableProgressBar { - bar = progressbar.StartNewByteBar(!bd.disableProgressBar, totalBytes) - defer bar.Finish() - } - +func (bd *BackupDestination) UploadPath(ctx context.Context, baseLocalPath string, files []string, remotePath string, RetriesOnFailure int, RetriesDuration time.Duration, maxSpeed uint64) (int64, error) { + totalBytes := int64(0) for _, filename := range files { + startTime := time.Now() + fInfo, err := os.Stat(filepath.Clean(path.Join(baseLocalPath, filename))) + if err != nil { + return 0, err + } + if fInfo.Mode().IsRegular() { + totalBytes += fInfo.Size() + } f, err := os.Open(filepath.Clean(path.Join(baseLocalPath, filename))) if err != nil { return 0, err @@ -589,19 +494,28 @@ func (bd *BackupDestination) UploadPath(ctx context.Context, size int64, baseLoc closeFile() return 0, err } - fi, err := f.Stat() - if err != nil { - return 0, err - } - if !bd.disableProgressBar { - bar.Add64(fi.Size()) - } closeFile() + bd.throttleSpeed(startTime, fInfo.Size(), maxSpeed) } return totalBytes, nil } +func (bd *BackupDestination) throttleSpeed(startTime time.Time, size int64, maxSpeed uint64) { + if maxSpeed > 0 && size > 0 { + timeSince := time.Since(startTime).Nanoseconds() + currentSpeed := uint64(size*1000000000) / uint64(timeSince) + if currentSpeed > maxSpeed { + + // Calculate how long to sleep to reduce the average speed to maxSpeed + excessSpeed := currentSpeed - maxSpeed + excessData := uint64(size) - (maxSpeed * uint64(timeSince) / 1000000000) + sleepTime := time.Duration((excessData*1000000000)/excessSpeed) * time.Nanosecond + time.Sleep(sleepTime) + } + } +} + func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhouse.ClickHouse, calcMaxSize bool, backupName string) (*BackupDestination, error) { var err error // https://github.com/Altinity/clickhouse-backup/issues/404 @@ -619,16 +533,25 @@ func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhous } switch cfg.General.RemoteStorage { case "azblob": - azblobStorage := &AzureBlob{Config: &cfg.AzureBlob} + azblobStorage := &AzureBlob{ + Config: &cfg.AzureBlob, + } azblobStorage.Config.Path, err = ch.ApplyMacros(ctx, azblobStorage.Config.Path) if err != nil { return nil, err } + azblobStorage.Config.ObjectDiskPath, err = ch.ApplyMacros(ctx, azblobStorage.Config.ObjectDiskPath) + if err != nil { + return nil, err + } bufferSize := azblobStorage.Config.BufferSize // https://github.com/Altinity/clickhouse-backup/issues/317 if bufferSize <= 0 { bufferSize = int(cfg.General.MaxFileSize) / cfg.AzureBlob.MaxPartsCount + if int(cfg.General.MaxFileSize)%cfg.AzureBlob.MaxPartsCount > 0 { + bufferSize += int(cfg.General.MaxFileSize) % cfg.AzureBlob.MaxPartsCount + } if bufferSize < 2*1024*1024 { bufferSize = 2 * 1024 * 1024 } @@ -641,12 +564,14 @@ func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhous azblobStorage, cfg.AzureBlob.CompressionFormat, cfg.AzureBlob.CompressionLevel, - cfg.General.DisableProgressBar, }, nil case "s3": partSize := cfg.S3.PartSize if cfg.S3.PartSize <= 0 { partSize = cfg.General.MaxFileSize / cfg.S3.MaxPartsCount + if cfg.General.MaxFileSize%cfg.S3.MaxPartsCount > 0 { + partSize++ + } if partSize < 5*1024*1024 { partSize = 5 * 1024 * 1024 } @@ -657,13 +582,17 @@ func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhous s3Storage := &S3{ Config: &cfg.S3, Concurrency: cfg.S3.Concurrency, - BufferSize: 512 * 1024, + BufferSize: 128 * 1024, PartSize: partSize, } s3Storage.Config.Path, err = ch.ApplyMacros(ctx, s3Storage.Config.Path) if err != nil { return nil, err } + s3Storage.Config.ObjectDiskPath, err = ch.ApplyMacros(ctx, s3Storage.Config.ObjectDiskPath) + if err != nil { + return nil, err + } // https://github.com/Altinity/clickhouse-backup/issues/588 if len(s3Storage.Config.ObjectLabels) > 0 && backupName != "" { objectLabels := s3Storage.Config.ObjectLabels @@ -677,7 +606,6 @@ func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhous s3Storage, cfg.S3.CompressionFormat, cfg.S3.CompressionLevel, - cfg.General.DisableProgressBar, }, nil case "gcs": googleCloudStorage := &GCS{Config: &cfg.GCS} @@ -685,6 +613,10 @@ func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhous if err != nil { return nil, err } + googleCloudStorage.Config.ObjectDiskPath, err = ch.ApplyMacros(ctx, googleCloudStorage.Config.ObjectDiskPath) + if err != nil { + return nil, err + } // https://github.com/Altinity/clickhouse-backup/issues/588 if len(googleCloudStorage.Config.ObjectLabels) > 0 && backupName != "" { objectLabels := googleCloudStorage.Config.ObjectLabels @@ -698,7 +630,6 @@ func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhous googleCloudStorage, cfg.GCS.CompressionFormat, cfg.GCS.CompressionLevel, - cfg.General.DisableProgressBar, }, nil case "cos": tencentStorage := &COS{Config: &cfg.COS} @@ -710,7 +641,6 @@ func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhous tencentStorage, cfg.COS.CompressionFormat, cfg.COS.CompressionLevel, - cfg.General.DisableProgressBar, }, nil case "ftp": ftpStorage := &FTP{ @@ -724,7 +654,6 @@ func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhous ftpStorage, cfg.FTP.CompressionFormat, cfg.FTP.CompressionLevel, - cfg.General.DisableProgressBar, }, nil case "sftp": sftpStorage := &SFTP{ @@ -738,14 +667,13 @@ func NewBackupDestination(ctx context.Context, cfg *config.Config, ch *clickhous sftpStorage, cfg.SFTP.CompressionFormat, cfg.SFTP.CompressionLevel, - cfg.General.DisableProgressBar, }, nil default: - return nil, fmt.Errorf("storage type '%s' is not supported", cfg.General.RemoteStorage) + return nil, fmt.Errorf("NewBackupDestination error: storage type '%s' is not supported", cfg.General.RemoteStorage) } } -// ApplyMacrosToObjectLabels - https://github.com/Altinity/clickhouse-backup/issues/588 +// ApplyMacrosToObjectLabels https://github.com/Altinity/clickhouse-backup/issues/588 func ApplyMacrosToObjectLabels(ctx context.Context, objectLabels map[string]string, ch *clickhouse.ClickHouse, backupName string) (map[string]string, error) { var err error for k, v := range objectLabels { diff --git a/pkg/storage/object_disk/object_disk.go b/pkg/storage/object_disk/object_disk.go index dbb40c20..2f43810a 100644 --- a/pkg/storage/object_disk/object_disk.go +++ b/pkg/storage/object_disk/object_disk.go @@ -11,13 +11,15 @@ import ( "path" "strconv" "strings" + "sync" "time" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/storage" + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/storage" "github.com/antchfx/xmlquery" - apexLog "github.com/apex/log" + "github.com/puzpuzpuz/xsync" + "github.com/rs/zerolog/log" ) type MetadataVersion uint32 @@ -189,7 +191,7 @@ type ObjectStorageCredentials struct { AzureContainerName string } -var DisksCredentials map[string]ObjectStorageCredentials +var DisksCredentials = xsync.NewMapOf[ObjectStorageCredentials]() type ObjectStorageConnection struct { Type string @@ -202,10 +204,10 @@ func (c *ObjectStorageConnection) GetRemoteStorage() storage.RemoteStorage { switch c.Type { case "s3": return c.S3 - case "azure_blob_storage": + case "azure", "azure_blob_storage": return c.AzureBlob } - apexLog.Fatalf("invalid ObjectStorageConnection.type %s", c.Type) + log.Fatal().Stack().Msgf("invalid ObjectStorageConnection.type %s", c.Type) return nil } @@ -213,10 +215,10 @@ func (c *ObjectStorageConnection) GetRemoteBucket() string { switch c.Type { case "s3": return c.S3.Config.Bucket - case "azure_blob_storage": + case "azure", "azure_blob_storage": return c.AzureBlob.Config.Container } - apexLog.Fatalf("invalid ObjectStorageConnection.type %s", c.Type) + log.Fatal().Stack().Msgf("invalid ObjectStorageConnection.type %s", c.Type) return "" } @@ -224,33 +226,34 @@ func (c *ObjectStorageConnection) GetRemotePath() string { switch c.Type { case "s3": return c.S3.Config.Path - case "azure_blob_storage": + case "azure", "azure_blob_storage": return c.AzureBlob.Config.Path } - apexLog.Fatalf("invalid ObjectStorageConnection.type %s", c.Type) + log.Fatal().Stack().Msgf("invalid ObjectStorageConnection.type %s", c.Type) return "" } -var DisksConnections map[string]ObjectStorageConnection -var SystemDisks map[string]clickhouse.Disk +var DisksConnections = xsync.NewMapOf[*ObjectStorageConnection]() + +var SystemDisks = xsync.NewMapOf[clickhouse.Disk]() + +var InitCredentialsAndConnectionsMutex sync.Mutex func InitCredentialsAndConnections(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName string) error { var err error - if _, exists := DisksCredentials[diskName]; !exists { - DisksCredentials, err = getObjectDisksCredentials(ctx, ch) - if err != nil { + InitCredentialsAndConnectionsMutex.Lock() + defer InitCredentialsAndConnectionsMutex.Unlock() + if _, exists := DisksCredentials.Load(diskName); !exists { + if err = getObjectDisksCredentials(ctx, ch); err != nil { return err } } - if _, exists := DisksConnections[diskName]; !exists { - if DisksConnections == nil { - DisksConnections = make(map[string]ObjectStorageConnection) - } + if _, exists := DisksConnections.Load(diskName); !exists { connection, err := makeObjectDiskConnection(ctx, ch, cfg, diskName) if err != nil { return err } - DisksConnections[diskName] = *connection + DisksConnections.Store(diskName, connection) } return nil } @@ -266,7 +269,7 @@ func ReadMetadataFromFile(path string) (*Metadata, error) { func ReadMetadataFromReader(metadataFile io.ReadCloser, path string) (*Metadata, error) { defer func() { if err := metadataFile.Close(); err != nil { - apexLog.Warnf("can't close reader %s: %v", path, err) + log.Warn().Msgf("can't close reader %s: %v", path, err) } }() @@ -285,29 +288,30 @@ func WriteMetadataToFile(metadata *Metadata, path string) error { } defer func() { if err = metadataFile.Close(); err != nil { - apexLog.Warnf("can't close %s: %v", path, err) + log.Warn().Msgf("can't close %s: %v", path, err) } }() return metadata.writeToFile(metadataFile) } -func getObjectDisksCredentials(ctx context.Context, ch *clickhouse.ClickHouse) (map[string]ObjectStorageCredentials, error) { - credentials := make(map[string]ObjectStorageCredentials) - if version, err := ch.GetVersion(ctx); err != nil { - return nil, err +func getObjectDisksCredentials(ctx context.Context, ch *clickhouse.ClickHouse) error { + var version int + var err error + if version, err = ch.GetVersion(ctx); err != nil { + return err } else if version <= 20006000 { - return credentials, nil + return nil } configFile, doc, err := ch.ParseXML(ctx, "config.xml") if err != nil { - return nil, err + return err } root := xmlquery.FindOne(doc, "/") disks := xmlquery.Find(doc, fmt.Sprintf("/%s/storage_configuration/disks/*", root.Data)) for _, d := range disks { diskName := d.Data if diskTypeNode := d.SelectElement("type"); diskTypeNode != nil { - diskType := diskTypeNode.InnerText() + diskType := strings.Trim(diskTypeNode.InnerText(), "\r\n \t") switch diskType { case "s3", "s3_plain": creds := ObjectStorageCredentials{ @@ -320,16 +324,21 @@ func getObjectDisksCredentials(ctx context.Context, ch *clickhouse.ClickHouse) ( } if endPointNode := d.SelectElement("endpoint"); endPointNode != nil { creds.EndPoint = strings.Trim(endPointNode.InnerText(), "\r\n \t") + // macros works only after 23.3+ https://github.com/Altinity/clickhouse-backup/issues/750 + if version > 23003000 { + if creds.EndPoint, err = ch.ApplyMacros(ctx, creds.EndPoint); err != nil { + return fmt.Errorf("%s -> /%s/storage_configuration/disks/%s apply macros to error: %v", configFile, root.Data, diskName, err) + } + } } else { - return nil, fmt.Errorf("%s -> /%s/storage_configuration/disks/%s doesn't contains ", configFile, root.Data, diskName) + return fmt.Errorf("%s -> /%s/storage_configuration/disks/%s doesn't contains ", configFile, root.Data, diskName) } if regionNode := d.SelectElement("region"); regionNode != nil { creds.S3Region = strings.Trim(regionNode.InnerText(), "\r\n \t") } + creds.S3StorageClass = "STANDARD" if storageClassNode := d.SelectElement("s3_storage_class"); storageClassNode != nil { creds.S3StorageClass = strings.Trim(storageClassNode.InnerText(), "\r\n \t") - } else { - creds.S3StorageClass = "STANDARD" } accessKeyNode := d.SelectElement("access_key_id") secretKeyNode := d.SelectElement("secret_access_key") @@ -338,40 +347,40 @@ func getObjectDisksCredentials(ctx context.Context, ch *clickhouse.ClickHouse) ( creds.S3AccessKey = strings.Trim(accessKeyNode.InnerText(), "\r\n \t") creds.S3SecretKey = strings.Trim(secretKeyNode.InnerText(), "\r\n \t") } else { - apexLog.Warnf("%s -> /%s/storage_configuration/disks/%s doesn't contains and environment variables will use", configFile, root.Data, diskName) + log.Warn().Msgf("%s -> /%s/storage_configuration/disks/%s doesn't contains and environment variables will use", configFile, root.Data, diskName) creds.S3AssumeRole = os.Getenv("AWS_ROLE_ARN") if useEnvironmentCredentials != nil { creds.S3AccessKey = os.Getenv("AWS_ACCESS_KEY_ID") creds.S3SecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") } } - credentials[diskName] = creds + DisksCredentials.Store(diskName, creds) break - case "azure_blob_storage": + case "azure", "azure_blob_storage": creds := ObjectStorageCredentials{ Type: "azblob", } accountUrlNode := d.SelectElement("storage_account_url") if accountUrlNode == nil { - return nil, fmt.Errorf("%s -> /%s/storage_configuration/disks/%s doesn't contains ", configFile, root.Data, diskName) + return fmt.Errorf("%s -> /%s/storage_configuration/disks/%s doesn't contains ", configFile, root.Data, diskName) } creds.EndPoint = strings.Trim(accountUrlNode.InnerText(), "\r\n \t") containerNameNode := d.SelectElement("container_name") if containerNameNode == nil { - return nil, fmt.Errorf("%s -> /%s/storage_configuration/disks/%s doesn't contains ", configFile, root.Data, diskName) + return fmt.Errorf("%s -> /%s/storage_configuration/disks/%s doesn't contains ", configFile, root.Data, diskName) } creds.AzureContainerName = strings.Trim(containerNameNode.InnerText(), "\r\n \t") accountNameNode := d.SelectElement("account_name") if containerNameNode == nil { - return nil, fmt.Errorf("%s -> /%s/storage_configuration/disks/%s doesn't contains ", configFile, root.Data, diskName) + return fmt.Errorf("%s -> /%s/storage_configuration/disks/%s doesn't contains ", configFile, root.Data, diskName) } creds.AzureAccountName = strings.Trim(accountNameNode.InnerText(), "\r\n \t") accountKeyNode := d.SelectElement("account_key") if containerNameNode == nil { - return nil, fmt.Errorf("%s -> /%s/storage_configuration/disks/%s doesn't contains ", configFile, root.Data, diskName) + return fmt.Errorf("%s -> /%s/storage_configuration/disks/%s doesn't contains ", configFile, root.Data, diskName) } creds.AzureAccountKey = strings.Trim(accountKeyNode.InnerText(), "\r\n \t") - credentials[diskName] = creds + DisksCredentials.Store(diskName, creds) break } } @@ -382,40 +391,43 @@ func getObjectDisksCredentials(ctx context.Context, ch *clickhouse.ClickHouse) ( diskType := diskTypeNode.InnerText() switch diskType { case "encrypted", "cache": - _, exists := credentials[diskName] + _, exists := DisksCredentials.Load(diskName) if !exists { if diskNode := d.SelectElement("disk"); diskNode != nil { childDiskName := diskNode.InnerText() - credentials[diskName] = credentials[childDiskName] + if childCreds, childExists := DisksCredentials.Load(childDiskName); childExists { + DisksCredentials.Store(diskName, childCreds) + } else { + log.Warn().Msgf("disk %s with type %s, reference to childDisk %s which not contains DiskCredentials", diskName, diskType, childDiskName) + } } } } } } - return credentials, nil + return nil } func makeObjectDiskConnection(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName string) (*ObjectStorageConnection, error) { - creds, exists := DisksCredentials[diskName] + creds, exists := DisksCredentials.Load(diskName) if !exists { return nil, fmt.Errorf("%s is not present in object_disk.DisksCredentials", diskName) } connection := ObjectStorageConnection{} - if SystemDisks == nil || len(SystemDisks) == 0 { + if SystemDisks.Size() == 0 { disks, err := ch.GetDisks(ctx, false) if err != nil { return nil, err } - SystemDisks = make(map[string]clickhouse.Disk, len(disks)) for _, d := range disks { - SystemDisks[d.Name] = d + SystemDisks.Store(d.Name, d) } } - disk, exists := SystemDisks[diskName] + disk, exists := SystemDisks.Load(diskName) if !exists { return nil, fmt.Errorf("%s is not presnet in object_disk.SystemDisks", diskName) } - if disk.Type != "s3" && disk.Type != "s3_plain" && disk.Type != "azure_blob_storage" { + if disk.Type != "s3" && disk.Type != "s3_plain" && disk.Type != "azure_blob_storage" && disk.Type != "azure" && disk.Type != "encrypted" { return nil, fmt.Errorf("%s have unsupported type %s", diskName, disk.Type) } connection.MetadataPath = disk.Path @@ -428,10 +440,12 @@ func makeObjectDiskConnection(ctx context.Context, ch *clickhouse.ClickHouse, cf if err != nil { return nil, err } - s3cfg.Endpoint = s3URL.Scheme + "://" + s3URL.Host if cfg.S3.Concurrency > 0 { s3cfg.Concurrency = cfg.S3.Concurrency } + if cfg.S3.DisableCertVerification && !strings.HasSuffix(s3URL.Host, ".amazonaws.com") && !strings.HasSuffix(s3URL.Host, ".googleapis.com") { + s3cfg.DisableCertVerification = cfg.S3.DisableCertVerification + } s3cfg.Region = "us-east-1" if creds.S3Region != "" { s3cfg.Region = creds.S3Region @@ -461,9 +475,10 @@ func makeObjectDiskConnection(ctx context.Context, ch *clickhouse.ClickHouse, cf if len(hostParts) >= 3 { s3cfg.Region = hostParts[2] } - s3cfg.Path = s3URL.Path + s3cfg.Path = strings.Trim(s3URL.Path, "/") s3cfg.ForcePathStyle = false } else { + s3cfg.Endpoint = s3URL.Scheme + "://" + s3URL.Host pathItems := strings.Split(strings.Trim(s3URL.Path, "/"), "/") s3cfg.Bucket = pathItems[0] s3cfg.Path = path.Join(pathItems[1:]...) @@ -471,6 +486,7 @@ func makeObjectDiskConnection(ctx context.Context, ch *clickhouse.ClickHouse, cf } // need for CopyObject s3cfg.ObjectDiskPath = s3cfg.Path + s3cfg.Debug = cfg.S3.Debug connection.S3 = &storage.S3{Config: &s3cfg} if err = connection.S3.Connect(ctx); err != nil { return nil, err @@ -509,6 +525,7 @@ func makeObjectDiskConnection(ctx context.Context, ch *clickhouse.ClickHouse, cf if creds.AzureContainerName != "" { azureCfg.Container = creds.AzureContainerName } + azureCfg.Debug = cfg.AzureBlob.Debug connection.AzureBlob = &storage.AzureBlob{Config: &azureCfg} if err = connection.AzureBlob.Connect(ctx); err != nil { return nil, err @@ -518,7 +535,7 @@ func makeObjectDiskConnection(ctx context.Context, ch *clickhouse.ClickHouse, cf } func ConvertLocalPathToRemote(diskName, localPath string) (string, error) { - connection, exists := DisksConnections[diskName] + connection, exists := DisksConnections.Load(diskName) if !exists { return "", fmt.Errorf("%s is not present in object_disk.DisksConnections", diskName) } @@ -533,7 +550,7 @@ func ConvertLocalPathToRemote(diskName, localPath string) (string, error) { } func GetFileReader(ctx context.Context, diskName, remotePath string) (io.ReadCloser, error) { - connection, exists := DisksConnections[diskName] + connection, exists := DisksConnections.Load(diskName) if !exists { return nil, fmt.Errorf("%s not exits in object_disk.DisksConnections", diskName) } @@ -564,7 +581,7 @@ func ReadFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config } func PutFile(ctx context.Context, diskName, remotePath string, content []byte) error { - connection, exists := DisksConnections[diskName] + connection, exists := DisksConnections.Load(diskName) if !exists { return fmt.Errorf("%s not exits in object_disk.DisksConnections", diskName) } @@ -587,7 +604,7 @@ func WriteFileContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *confi } func DeleteFile(ctx context.Context, diskName, remotePath string) error { - connection, exists := DisksConnections[diskName] + connection, exists := DisksConnections.Load(diskName) if !exists { return fmt.Errorf("%s not exits in object_disk.DisksConnections", diskName) } @@ -595,6 +612,7 @@ func DeleteFile(ctx context.Context, diskName, remotePath string) error { return remoteStorage.DeleteFile(ctx, remotePath) } +/* func DeleteFileWithContent(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName, localPath string) error { if err := InitCredentialsAndConnections(ctx, ch, cfg, diskName); err != nil { return err @@ -622,13 +640,10 @@ func GetFileSize(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Con } return fileInfo.Size(), nil } +*/ -func CopyObject(ctx context.Context, ch *clickhouse.ClickHouse, cfg *config.Config, diskName, srcBucket, srcKey, dstPath string) error { - if err := InitCredentialsAndConnections(ctx, ch, cfg, diskName); err != nil { - return err - } - connection := DisksConnections[diskName] +func CopyObject(ctx context.Context, diskName string, srcSize int64, srcBucket, srcKey, dstPath string) (int64, error) { + connection, _ := DisksConnections.Load(diskName) remoteStorage := connection.GetRemoteStorage() - _, err := remoteStorage.CopyObject(ctx, srcBucket, srcKey, dstPath) - return err + return remoteStorage.CopyObject(ctx, srcSize, srcBucket, srcKey, dstPath) } diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go index 3a8f6a74..fa73c4ac 100644 --- a/pkg/storage/s3.go +++ b/pkg/storage/s3.go @@ -4,16 +4,16 @@ import ( "context" "crypto/tls" "fmt" - apexLog "github.com/apex/log" "io" "net/http" "os" "path" + "sort" "strings" "sync" "time" - "github.com/Altinity/clickhouse-backup/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" "github.com/aws/aws-sdk-go-v2/aws" v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" awsV2Config "github.com/aws/aws-sdk-go-v2/config" @@ -30,7 +30,6 @@ import ( "github.com/rs/zerolog" "github.com/rs/zerolog/log" "golang.org/x/sync/errgroup" - "golang.org/x/sync/semaphore" ) type S3LogToZeroLogAdapter struct { @@ -116,22 +115,19 @@ func (s *S3) Connect(ctx context.Context) error { if s.Config.Region != "" { awsConfig.Region = s.Config.Region } + // AWS IRSA handling, look https://github.com/Altinity/clickhouse-backup/issues/798 awsRoleARN := os.Getenv("AWS_ROLE_ARN") - if s.Config.AssumeRoleARN != "" || awsRoleARN != "" { - stsClient := sts.NewFromConfig(awsConfig) - if awsRoleARN != "" { - awsConfig.Credentials = stscreds.NewAssumeRoleProvider(stsClient, awsRoleARN) - } else { - awsConfig.Credentials = stscreds.NewAssumeRoleProvider(stsClient, s.Config.AssumeRoleARN) - } - } - awsWebIdentityTokenFile := os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE") + stsClient := sts.NewFromConfig(awsConfig) if awsRoleARN != "" && awsWebIdentityTokenFile != "" { - stsClient := sts.NewFromConfig(awsConfig) awsConfig.Credentials = stscreds.NewWebIdentityRoleProvider( stsClient, awsRoleARN, stscreds.IdentityTokenFile(awsWebIdentityTokenFile), ) + } else if s.Config.AssumeRoleARN != "" { + // backup role S3_ASSUME_ROLE_ARN have high priority than AWS_ROLE_ARN see https://github.com/Altinity/clickhouse-backup/issues/898 + awsConfig.Credentials = stscreds.NewAssumeRoleProvider(stsClient, s.Config.AssumeRoleARN) + } else if awsRoleARN != "" { + awsConfig.Credentials = stscreds.NewAssumeRoleProvider(stsClient, awsRoleARN) } if s.Config.AccessKey != "" && s.Config.SecretKey != "" { @@ -171,7 +167,9 @@ func (s *S3) Connect(ctx context.Context) error { // allow GCS over S3, remove Accept-Encoding header from sign https://stackoverflow.com/a/74382598/1204665, https://github.com/aws/aws-sdk-go-v2/issues/1816 if strings.Contains(s.Config.Endpoint, "storage.googleapis.com") { // Assign custom client with our own transport - awsConfig.HTTPClient = &http.Client{Transport: &RecalculateV4Signature{httpTransport, v4.NewSigner(), awsConfig}} + awsConfig.HTTPClient = &http.Client{Transport: &RecalculateV4Signature{httpTransport, v4.NewSigner(func(signer *v4.SignerOptions) { + signer.DisableURIPathEscaping = true + }), awsConfig}} } s.client = s3.NewFromConfig(awsConfig, func(o *s3.Options) { o.UsePathStyle = s.Config.ForcePathStyle @@ -198,11 +196,16 @@ func (s *S3) Close(ctx context.Context) error { } func (s *S3) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error) { - getRequest := &s3.GetObjectInput{ + return s.GetFileReaderAbsolute(ctx, path.Join(s.Config.Path, key)) +} + +func (s *S3) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error) { + params := &s3.GetObjectInput{ Bucket: aws.String(s.Config.Bucket), - Key: aws.String(path.Join(s.Config.Path, key)), + Key: aws.String(key), } - resp, err := s.client.GetObject(ctx, getRequest) + s.enrichGetObjectParams(params) + resp, err := s.client.GetObject(ctx, params) if err != nil { var opError *smithy.OperationError if errors.As(err, &opError) { @@ -216,7 +219,7 @@ func (s *S3) GetFileReader(ctx context.Context, key string) (io.ReadCloser, erro log.Warn().Msgf("restoreObject %s, return error: %v", key, restoreErr) return nil, err } - if resp, err = s.client.GetObject(ctx, getRequest); err != nil { + if resp, err = s.client.GetObject(ctx, params); err != nil { log.Warn().Msgf("second GetObject %s, return error: %v", key, err) return nil, err } @@ -231,6 +234,21 @@ func (s *S3) GetFileReader(ctx context.Context, key string) (io.ReadCloser, erro return resp.Body, nil } +func (s *S3) enrichGetObjectParams(params *s3.GetObjectInput) { + if s.Config.SSECustomerAlgorithm != "" { + params.SSECustomerAlgorithm = aws.String(s.Config.SSECustomerAlgorithm) + } + if s.Config.SSECustomerKey != "" { + params.SSECustomerKey = aws.String(s.Config.SSECustomerKey) + } + if s.Config.SSECustomerKeyMD5 != "" { + params.SSECustomerKeyMD5 = aws.String(s.Config.SSECustomerKeyMD5) + } + if s.Config.RequestPayer != "" { + params.RequestPayer = s3types.RequestPayer(s.Config.RequestPayer) + } +} + func (s *S3) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string) (io.ReadCloser, error) { /* unfortunately, multipart download require allocate additional disk space and don't allow us to decompress data directly from stream */ @@ -253,13 +271,24 @@ func (s *S3) GetFileReaderWithLocalPath(ctx context.Context, key, localPath stri } func (s *S3) PutFile(ctx context.Context, key string, r io.ReadCloser) error { + return s.PutFileAbsolute(ctx, path.Join(s.Config.Path, key), r) +} + +func (s *S3) PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error { params := s3.PutObjectInput{ - ACL: s3types.ObjectCannedACL(s.Config.ACL), Bucket: aws.String(s.Config.Bucket), - Key: aws.String(path.Join(s.Config.Path, key)), + Key: aws.String(key), Body: r, StorageClass: s3types.StorageClass(strings.ToUpper(s.Config.StorageClass)), } + if s.Config.CheckSumAlgorithm != "" { + params.ChecksumAlgorithm = s3types.ChecksumAlgorithm(s.Config.CheckSumAlgorithm) + } + + // ACL shall be optional, fix https://github.com/Altinity/clickhouse-backup/issues/785 + if s.Config.ACL != "" { + params.ACL = s3types.ObjectCannedACL(s.Config.ACL) + } // https://github.com/Altinity/clickhouse-backup/issues/588 if len(s.Config.ObjectLabels) > 0 { tags := "" @@ -298,15 +327,18 @@ func (s *S3) deleteKey(ctx context.Context, key string) error { Bucket: aws.String(s.Config.Bucket), Key: aws.String(key), } + if s.Config.RequestPayer != "" { + params.RequestPayer = s3types.RequestPayer(s.Config.RequestPayer) + } if s.versioning { objVersion, err := s.getObjectVersion(ctx, key) if err != nil { - return errors.Wrapf(err, "deleteKey, obtaining object version %+v", params) + return errors.Wrapf(err, "deleteKey, obtaining object version bucket: %s key: %s", s.Config.Bucket, key) } params.VersionId = objVersion } if _, err := s.client.DeleteObject(ctx, params); err != nil { - return errors.Wrapf(err, "deleteKey, deleting object %+v", params) + return errors.Wrapf(err, "deleteKey, deleting object bucket: %s key: %s version: %v", s.Config.Bucket, key, params.VersionId) } return nil } @@ -334,7 +366,10 @@ func (s *S3) isVersioningEnabled(ctx context.Context) bool { func (s *S3) getObjectVersion(ctx context.Context, key string) (*string, error) { params := &s3.HeadObjectInput{ Bucket: aws.String(s.Config.Bucket), - Key: aws.String(path.Join(s.Config.Path, key)), + Key: aws.String(key), + } + if s.Config.RequestPayer != "" { + params.RequestPayer = s3types.RequestPayer(s.Config.RequestPayer) } object, err := s.client.HeadObject(ctx, params) if err != nil { @@ -344,10 +379,12 @@ func (s *S3) getObjectVersion(ctx context.Context, key string) (*string, error) } func (s *S3) StatFile(ctx context.Context, key string) (RemoteFile, error) { - head, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ + params := &s3.HeadObjectInput{ Bucket: aws.String(s.Config.Bucket), Key: aws.String(path.Join(s.Config.Path, key)), - }) + } + s.enrichHeadParams(params) + head, err := s.client.HeadObject(ctx, params) if err != nil { var opError *smithy.OperationError if errors.As(err, &opError) { @@ -360,35 +397,40 @@ func (s *S3) StatFile(ctx context.Context, key string) (RemoteFile, error) { } return nil, err } - return &s3File{head.ContentLength, *head.LastModified, string(head.StorageClass), key}, nil + return &s3File{*head.ContentLength, *head.LastModified, string(head.StorageClass), key}, nil } func (s *S3) Walk(ctx context.Context, s3Path string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error { + prefix := path.Join(s.Config.Path, s3Path) + return s.WalkAbsolute(ctx, prefix, recursive, process) +} + +func (s *S3) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(ctx context.Context, r RemoteFile) error) error { g, ctx := errgroup.WithContext(ctx) s3Files := make(chan *s3File) g.Go(func() error { defer close(s3Files) - return s.remotePager(ctx, path.Join(s.Config.Path, s3Path), recursive, func(page *s3.ListObjectsV2Output) { + return s.remotePager(ctx, prefix, recursive, func(page *s3.ListObjectsV2Output) { for _, cp := range page.CommonPrefixes { s3Files <- &s3File{ - name: strings.TrimPrefix(*cp.Prefix, path.Join(s.Config.Path, s3Path)), + name: strings.TrimPrefix(*cp.Prefix, prefix), } } for _, c := range page.Contents { s3Files <- &s3File{ - c.Size, + *c.Size, *c.LastModified, string(c.StorageClass), - strings.TrimPrefix(*c.Key, path.Join(s.Config.Path, s3Path)), + strings.TrimPrefix(*c.Key, prefix), } } }) }) g.Go(func() error { var err error - for s3File := range s3Files { + for s3FileItem := range s3Files { if err == nil { - err = process(ctx, s3File) + err = process(ctx, s3FileItem) } } return err @@ -403,7 +445,7 @@ func (s *S3) remotePager(ctx context.Context, s3Path string, recursive bool, pro } params := &s3.ListObjectsV2Input{ Bucket: aws.String(s.Config.Bucket), // Required - MaxKeys: 1000, + MaxKeys: aws.Int32(1000), Prefix: aws.String(prefix), } if !recursive { @@ -422,105 +464,34 @@ func (s *S3) remotePager(ctx context.Context, s3Path string, recursive bool, pro return nil } -func (s *S3) CopyObject(ctx context.Context, srcBucket, srcKey, dstKey string) (int64, error) { +func (s *S3) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error) { dstKey = path.Join(s.Config.ObjectDiskPath, dstKey) - if strings.Contains(s.Config.Endpoint, "storage.googleapis.com") { - params := s3.CopyObjectInput{ + log.Debug().Msgf("S3->CopyObject %s/%s -> %s/%s", srcBucket, srcKey, s.Config.Bucket, dstKey) + // just copy object without multipart + if srcSize < 5*1024*1024*1024 || strings.Contains(s.Config.Endpoint, "storage.googleapis.com") { + params := &s3.CopyObjectInput{ Bucket: aws.String(s.Config.Bucket), Key: aws.String(dstKey), CopySource: aws.String(path.Join(srcBucket, srcKey)), StorageClass: s3types.StorageClass(strings.ToUpper(s.Config.StorageClass)), } - // https://github.com/Altinity/clickhouse-backup/issues/588 - if len(s.Config.ObjectLabels) > 0 { - tags := "" - for k, v := range s.Config.ObjectLabels { - if tags != "" { - tags += "&" - } - tags += k + "=" + v - } - params.Tagging = aws.String(tags) - } - if s.Config.SSE != "" { - params.ServerSideEncryption = s3types.ServerSideEncryption(s.Config.SSE) - } - if s.Config.SSEKMSKeyId != "" { - params.SSEKMSKeyId = aws.String(s.Config.SSEKMSKeyId) - } - if s.Config.SSECustomerAlgorithm != "" { - params.SSECustomerAlgorithm = aws.String(s.Config.SSECustomerAlgorithm) - } - if s.Config.SSECustomerKey != "" { - params.SSECustomerKey = aws.String(s.Config.SSECustomerKey) - } - if s.Config.SSECustomerKeyMD5 != "" { - params.SSECustomerKeyMD5 = aws.String(s.Config.SSECustomerKeyMD5) - } - if s.Config.SSEKMSEncryptionContext != "" { - params.SSEKMSEncryptionContext = aws.String(s.Config.SSEKMSEncryptionContext) - } - _, err := s.client.CopyObject(ctx, ¶ms) - if err != nil { - return 0, err - } - dstObjResp, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(s.Config.Bucket), - Key: aws.String(dstKey), - }) + s.enrichCopyObjectParams(params) + _, err := s.client.CopyObject(ctx, params) if err != nil { - return 0, err + return 0, fmt.Errorf("S3->CopyObject %s/%s -> %s/%s return error: %v", srcBucket, srcKey, s.Config.Bucket, dstKey, err) } - return dstObjResp.ContentLength, nil + return srcSize, nil } - // Get the size of the source object - sourceObjResp, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: aws.String(srcBucket), - Key: aws.String(srcKey), - }) - if err != nil { - return 0, err - } - srcSize := sourceObjResp.ContentLength // Initiate a multipart upload - params := s3.CreateMultipartUploadInput{ + createMultipartUploadParams := &s3.CreateMultipartUploadInput{ Bucket: aws.String(s.Config.Bucket), Key: aws.String(dstKey), StorageClass: s3types.StorageClass(strings.ToUpper(s.Config.StorageClass)), } - // https://github.com/Altinity/clickhouse-backup/issues/588 - if len(s.Config.ObjectLabels) > 0 { - tags := "" - for k, v := range s.Config.ObjectLabels { - if tags != "" { - tags += "&" - } - tags += k + "=" + v - } - params.Tagging = aws.String(tags) - } - if s.Config.SSE != "" { - params.ServerSideEncryption = s3types.ServerSideEncryption(s.Config.SSE) - } - if s.Config.SSEKMSKeyId != "" { - params.SSEKMSKeyId = aws.String(s.Config.SSEKMSKeyId) - } - if s.Config.SSECustomerAlgorithm != "" { - params.SSECustomerAlgorithm = aws.String(s.Config.SSECustomerAlgorithm) - } - if s.Config.SSECustomerKey != "" { - params.SSECustomerKey = aws.String(s.Config.SSECustomerKey) - } - if s.Config.SSECustomerKeyMD5 != "" { - params.SSECustomerKeyMD5 = aws.String(s.Config.SSECustomerKeyMD5) - } - if s.Config.SSEKMSEncryptionContext != "" { - params.SSEKMSEncryptionContext = aws.String(s.Config.SSEKMSEncryptionContext) - } - - initResp, err := s.client.CreateMultipartUpload(ctx, ¶ms) + s.enrichCreateMultipartUploadParams(createMultipartUploadParams) + initResp, err := s.client.CreateMultipartUpload(ctx, createMultipartUploadParams) if err != nil { - return 0, err + return 0, fmt.Errorf("S3->CopyObject %s/%s -> %s/%s, CreateMultipartUpload return error: %v", srcBucket, srcKey, s.Config.Bucket, dstKey, err) } // Get the upload ID @@ -528,25 +499,29 @@ func (s *S3) CopyObject(ctx context.Context, srcBucket, srcKey, dstKey string) ( // Set the part size (e.g., 5 MB) partSize := srcSize / s.Config.MaxPartsCount - if partSize < 5*1024*1024 { - partSize = 5 * 1014 * 1024 + if srcSize%s.Config.MaxPartsCount > 0 { + partSize++ + } + // 128Mb part size recommendation from https://repost.aws/questions/QUtW2_XaALTK63wv9XLSywiQ/s3-sync-command-is-slow-to-start-on-some-data + if partSize < 128*1024*1024 { + partSize = 128 * 1024 * 1024 + } + + if partSize > 5*1024*1024*1024 { + partSize = 5 * 1024 * 1024 * 1024 } // Calculate the number of parts numParts := (srcSize + partSize - 1) / partSize - copyPartSemaphore := semaphore.NewWeighted(int64(s.Config.Concurrency)) copyPartErrGroup, ctx := errgroup.WithContext(ctx) + copyPartErrGroup.SetLimit(s.Config.Concurrency * s.Config.Concurrency) var mu sync.Mutex var parts []s3types.CompletedPart // Copy each part of the object for partNumber := int64(1); partNumber <= numParts; partNumber++ { - if err := copyPartSemaphore.Acquire(ctx, 1); err != nil { - apexLog.Errorf("can't acquire semaphore during CopyObject data parts: %v", err) - break - } // Calculate the byte range for the part start := (partNumber - 1) * partSize end := partNumber * partSize @@ -556,76 +531,173 @@ func (s *S3) CopyObject(ctx context.Context, srcBucket, srcKey, dstKey string) ( currentPartNumber := int32(partNumber) copyPartErrGroup.Go(func() error { - defer copyPartSemaphore.Release(1) // Copy the part - partResp, err := s.client.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + uploadPartParams := &s3.UploadPartCopyInput{ Bucket: aws.String(s.Config.Bucket), Key: aws.String(dstKey), CopySource: aws.String(srcBucket + "/" + srcKey), CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", start, end-1)), UploadId: uploadID, - PartNumber: currentPartNumber, - }) + PartNumber: aws.Int32(currentPartNumber), + } + if s.Config.RequestPayer != "" { + uploadPartParams.RequestPayer = s3types.RequestPayer(s.Config.RequestPayer) + } + partResp, err := s.client.UploadPartCopy(ctx, uploadPartParams) if err != nil { - return err + return fmt.Errorf("S3->CopyObject %s/%s -> %s/%s, UploadPartCopy start=%d, end=%d return error: %v", srcBucket, srcKey, s.Config.Bucket, dstKey, start, end-1, err) } mu.Lock() - defer mu.Unlock() parts = append(parts, s3types.CompletedPart{ - ETag: partResp.CopyPartResult.ETag, - PartNumber: currentPartNumber, + ETag: partResp.CopyPartResult.ETag, + PartNumber: aws.Int32(currentPartNumber), + ChecksumCRC32: partResp.CopyPartResult.ChecksumCRC32, + ChecksumCRC32C: partResp.CopyPartResult.ChecksumCRC32C, + ChecksumSHA1: partResp.CopyPartResult.ChecksumSHA1, + ChecksumSHA256: partResp.CopyPartResult.ChecksumSHA256, }) + mu.Unlock() return nil }) } - if err := copyPartErrGroup.Wait(); err != nil { - _, abortErr := s.client.AbortMultipartUpload(context.Background(), &s3.AbortMultipartUploadInput{ + if wgWaitErr := copyPartErrGroup.Wait(); wgWaitErr != nil { + abortParams := &s3.AbortMultipartUploadInput{ Bucket: aws.String(s.Config.Bucket), Key: aws.String(dstKey), UploadId: uploadID, - }) + } + if s.Config.RequestPayer != "" { + abortParams.RequestPayer = s3types.RequestPayer(s.Config.RequestPayer) + } + _, abortErr := s.client.AbortMultipartUpload(context.Background(), abortParams) if abortErr != nil { - return 0, fmt.Errorf("aborting CopyObject multipart upload: %v, original error was: %v", abortErr, err) + return 0, fmt.Errorf("aborting CopyObject multipart upload: %v, original error was: %v", abortErr, wgWaitErr) } - return 0, fmt.Errorf("one of CopyObject go-routine return error: %v", err) + return 0, fmt.Errorf("one of CopyObject/Multipart go-routine return error: %v", wgWaitErr) } - + // Parts must be ordered by part number. + sort.Slice(parts, func(i int, j int) bool { + return *parts[i].PartNumber < *parts[j].PartNumber + }) // Complete the multipart upload - _, err = s.client.CompleteMultipartUpload(context.Background(), &s3.CompleteMultipartUploadInput{ + completeMultipartUploadParams := &s3.CompleteMultipartUploadInput{ Bucket: aws.String(s.Config.Bucket), Key: aws.String(dstKey), UploadId: uploadID, MultipartUpload: &s3types.CompletedMultipartUpload{Parts: parts}, - }) + } + if s.Config.RequestPayer != "" { + completeMultipartUploadParams.RequestPayer = s3types.RequestPayer(s.Config.RequestPayer) + } + _, err = s.client.CompleteMultipartUpload(context.Background(), completeMultipartUploadParams) if err != nil { return 0, fmt.Errorf("complete CopyObject multipart upload: %v", err) } - log.Debug().Msgf("S3->CopyObject %s/%s -> %s/%s", srcBucket, srcKey, s.Config.Bucket, dstKey) return srcSize, nil } +func (s *S3) enrichCreateMultipartUploadParams(params *s3.CreateMultipartUploadInput) { + if s.Config.CheckSumAlgorithm != "" { + params.ChecksumAlgorithm = s3types.ChecksumAlgorithm(s.Config.CheckSumAlgorithm) + } + if s.Config.RequestPayer != "" { + params.RequestPayer = s3types.RequestPayer(s.Config.RequestPayer) + } + // https://github.com/Altinity/clickhouse-backup/issues/588 + if len(s.Config.ObjectLabels) > 0 { + tags := "" + for k, v := range s.Config.ObjectLabels { + if tags != "" { + tags += "&" + } + tags += k + "=" + v + } + params.Tagging = aws.String(tags) + } + if s.Config.SSE != "" { + params.ServerSideEncryption = s3types.ServerSideEncryption(s.Config.SSE) + } + if s.Config.SSEKMSKeyId != "" { + params.SSEKMSKeyId = aws.String(s.Config.SSEKMSKeyId) + } + if s.Config.SSECustomerAlgorithm != "" { + params.SSECustomerAlgorithm = aws.String(s.Config.SSECustomerAlgorithm) + } + if s.Config.SSECustomerKey != "" { + params.SSECustomerKey = aws.String(s.Config.SSECustomerKey) + } + if s.Config.SSECustomerKeyMD5 != "" { + params.SSECustomerKeyMD5 = aws.String(s.Config.SSECustomerKeyMD5) + } + if s.Config.SSEKMSEncryptionContext != "" { + params.SSEKMSEncryptionContext = aws.String(s.Config.SSEKMSEncryptionContext) + } +} + +func (s *S3) enrichCopyObjectParams(params *s3.CopyObjectInput) { + if s.Config.CheckSumAlgorithm != "" { + params.ChecksumAlgorithm = s3types.ChecksumAlgorithm(s.Config.CheckSumAlgorithm) + } + // https://github.com/Altinity/clickhouse-backup/issues/588 + if len(s.Config.ObjectLabels) > 0 { + tags := "" + for k, v := range s.Config.ObjectLabels { + if tags != "" { + tags += "&" + } + tags += k + "=" + v + } + params.Tagging = aws.String(tags) + } + if s.Config.SSE != "" { + params.ServerSideEncryption = s3types.ServerSideEncryption(s.Config.SSE) + } + if s.Config.SSEKMSKeyId != "" { + params.SSEKMSKeyId = aws.String(s.Config.SSEKMSKeyId) + } + if s.Config.SSECustomerAlgorithm != "" { + params.SSECustomerAlgorithm = aws.String(s.Config.SSECustomerAlgorithm) + } + if s.Config.SSECustomerKey != "" { + params.SSECustomerKey = aws.String(s.Config.SSECustomerKey) + } + if s.Config.SSECustomerKeyMD5 != "" { + params.SSECustomerKeyMD5 = aws.String(s.Config.SSECustomerKeyMD5) + } + if s.Config.SSEKMSEncryptionContext != "" { + params.SSEKMSEncryptionContext = aws.String(s.Config.SSEKMSEncryptionContext) + } + if s.Config.RequestPayer != "" { + params.RequestPayer = s3types.RequestPayer(s.Config.RequestPayer) + } +} + func (s *S3) restoreObject(ctx context.Context, key string) error { - restoreRequest := s3.RestoreObjectInput{ + restoreRequest := &s3.RestoreObjectInput{ Bucket: aws.String(s.Config.Bucket), Key: aws.String(path.Join(s.Config.Path, key)), RestoreRequest: &s3types.RestoreRequest{ - Days: 1, + Days: aws.Int32(1), GlacierJobParameters: &s3types.GlacierJobParameters{ Tier: s3types.Tier("Expedited"), }, }, } - _, err := s.client.RestoreObject(ctx, &restoreRequest) + if s.Config.RequestPayer != "" { + restoreRequest.RequestPayer = s3types.RequestPayer(s.Config.RequestPayer) + } + _, err := s.client.RestoreObject(ctx, restoreRequest) if err != nil { return err } i := 0 for { - headObjectRequest := &s3.HeadObjectInput{ + restoreHeadParams := &s3.HeadObjectInput{ Bucket: aws.String(s.Config.Bucket), Key: aws.String(path.Join(s.Config.Path, key)), } - res, err := s.client.HeadObject(ctx, headObjectRequest) + s.enrichHeadParams(restoreHeadParams) + res, err := s.client.HeadObject(ctx, restoreHeadParams) if err != nil { return fmt.Errorf("restoreObject: failed to head %s object metadata, %v", path.Join(s.Config.Path, key), err) } @@ -640,6 +712,21 @@ func (s *S3) restoreObject(ctx context.Context, key string) error { } } +func (s *S3) enrichHeadParams(headParams *s3.HeadObjectInput) { + if s.Config.RequestPayer != "" { + headParams.RequestPayer = s3types.RequestPayer(s.Config.RequestPayer) + } + if s.Config.SSECustomerAlgorithm != "" { + headParams.SSECustomerAlgorithm = aws.String(s.Config.SSECustomerAlgorithm) + } + if s.Config.SSECustomerKey != "" { + headParams.SSECustomerKey = aws.String(s.Config.SSECustomerKey) + } + if s.Config.SSECustomerKeyMD5 != "" { + headParams.SSECustomerKeyMD5 = aws.String(s.Config.SSECustomerKeyMD5) + } +} + type s3File struct { size int64 lastModified time.Time diff --git a/pkg/storage/sftp.go b/pkg/storage/sftp.go index 6a6015ad..fba1ee3b 100644 --- a/pkg/storage/sftp.go +++ b/pkg/storage/sftp.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" "io" "os" "path" @@ -166,11 +166,15 @@ func (sftp *SFTP) DeleteDirectory(ctx context.Context, dirPath string) error { } func (sftp *SFTP) Walk(ctx context.Context, remotePath string, recursive bool, process func(context.Context, RemoteFile) error) error { - dir := path.Join(sftp.Config.Path, remotePath) - sftp.Debug("[SFTP_DEBUG] Walk %s, recursive=%v", dir, recursive) + prefix := path.Join(sftp.Config.Path, remotePath) + return sftp.WalkAbsolute(ctx, prefix, recursive, process) +} + +func (sftp *SFTP) WalkAbsolute(ctx context.Context, prefix string, recursive bool, process func(context.Context, RemoteFile) error) error { + sftp.Debug("[SFTP_DEBUG] Walk %s, recursive=%v", prefix, recursive) if recursive { - walker := sftp.sftpClient.Walk(dir) + walker := sftp.sftpClient.Walk(prefix) for walker.Step() { if err := walker.Err(); err != nil { return err @@ -179,7 +183,7 @@ func (sftp *SFTP) Walk(ctx context.Context, remotePath string, recursive bool, p if entry == nil { continue } - relName, _ := filepath.Rel(dir, walker.Path()) + relName, _ := filepath.Rel(prefix, walker.Path()) err := process(ctx, &sftpFile{ size: entry.Size(), lastModified: entry.ModTime(), @@ -190,9 +194,9 @@ func (sftp *SFTP) Walk(ctx context.Context, remotePath string, recursive bool, p } } } else { - entries, err := sftp.sftpClient.ReadDir(dir) + entries, err := sftp.sftpClient.ReadDir(prefix) if err != nil { - sftp.Debug("[SFTP_DEBUG] Walk::NonRecursive::ReadDir %s return error %v", dir, err) + sftp.Debug("[SFTP_DEBUG] Walk::NonRecursive::ReadDir %s return error %v", prefix, err) return err } for _, entry := range entries { @@ -210,8 +214,11 @@ func (sftp *SFTP) Walk(ctx context.Context, remotePath string, recursive bool, p } func (sftp *SFTP) GetFileReader(ctx context.Context, key string) (io.ReadCloser, error) { - filePath := path.Join(sftp.Config.Path, key) - return sftp.sftpClient.OpenFile(filePath, syscall.O_RDWR) + return sftp.GetFileReaderAbsolute(ctx, path.Join(sftp.Config.Path, key)) +} + +func (sftp *SFTP) GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error) { + return sftp.sftpClient.OpenFile(key, syscall.O_RDWR) } func (sftp *SFTP) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) (io.ReadCloser, error) { @@ -219,17 +226,20 @@ func (sftp *SFTP) GetFileReaderWithLocalPath(ctx context.Context, key, _ string) } func (sftp *SFTP) PutFile(ctx context.Context, key string, localFile io.ReadCloser) error { - filePath := path.Join(sftp.Config.Path, key) - if err := sftp.sftpClient.MkdirAll(path.Dir(filePath)); err != nil { - log.Warn().Msgf("sftp.sftpClient.MkdirAll(%s) err=%v", path.Dir(filePath), err) + return sftp.PutFileAbsolute(ctx, path.Join(sftp.Config.Path, key), localFile) +} + +func (sftp *SFTP) PutFileAbsolute(ctx context.Context, key string, localFile io.ReadCloser) error { + if err := sftp.sftpClient.MkdirAll(path.Dir(key)); err != nil { + log.Warn().Msgf("sftp.sftpClient.MkdirAll(%s) err=%v", path.Dir(key), err) } - remoteFile, err := sftp.sftpClient.Create(filePath) + remoteFile, err := sftp.sftpClient.Create(key) if err != nil { return err } defer func() { if err := remoteFile.Close(); err != nil { - log.Warn().Msgf("can't close %s err=%v", filePath, err) + log.Warn().Msgf("can't close %s err=%v", key, err) } }() if _, err = remoteFile.ReadFrom(localFile); err != nil { @@ -238,7 +248,7 @@ func (sftp *SFTP) PutFile(ctx context.Context, key string, localFile io.ReadClos return nil } -func (sftp *SFTP) CopyObject(ctx context.Context, srcBucket, srcKey, dstKey string) (int64, error) { +func (sftp *SFTP) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error) { return 0, fmt.Errorf("CopyObject not imlemented for %s", sftp.Kind()) } diff --git a/pkg/storage/structs.go b/pkg/storage/structs.go index d26b4c35..f90cf0eb 100644 --- a/pkg/storage/structs.go +++ b/pkg/storage/structs.go @@ -28,8 +28,11 @@ type RemoteStorage interface { DeleteFile(ctx context.Context, key string) error DeleteFileFromObjectDiskBackup(ctx context.Context, key string) error Walk(ctx context.Context, prefix string, recursive bool, fn func(context.Context, RemoteFile) error) error + WalkAbsolute(ctx context.Context, absolutePrefix string, recursive bool, fn func(context.Context, RemoteFile) error) error GetFileReader(ctx context.Context, key string) (io.ReadCloser, error) + GetFileReaderAbsolute(ctx context.Context, key string) (io.ReadCloser, error) GetFileReaderWithLocalPath(ctx context.Context, key, localPath string) (io.ReadCloser, error) PutFile(ctx context.Context, key string, r io.ReadCloser) error - CopyObject(ctx context.Context, srcBucket, srcKey, dstKey string) (int64, error) + PutFileAbsolute(ctx context.Context, key string, r io.ReadCloser) error + CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, dstKey string) (int64, error) } diff --git a/pkg/storage/utils.go b/pkg/storage/utils.go index 68743084..3ffada57 100644 --- a/pkg/storage/utils.go +++ b/pkg/storage/utils.go @@ -10,7 +10,7 @@ import ( "time" ) -func GetBackupsToDelete(backups []Backup, keep int) []Backup { +func GetBackupsToDeleteRemote(backups []Backup, keep int) []Backup { if len(backups) > keep { // sort backup ascending sort.SliceStable(backups, func(i, j int) bool { diff --git a/pkg/storage/utils_test.go b/pkg/storage/utils_test.go index 4a96137b..6f23093a 100644 --- a/pkg/storage/utils_test.go +++ b/pkg/storage/utils_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/Altinity/clickhouse-backup/pkg/metadata" + "github.com/Altinity/clickhouse-backup/v2/pkg/metadata" "github.com/stretchr/testify/assert" ) @@ -19,109 +19,109 @@ func timeParse(s string) time.Time { func TestGetBackupsToDelete(t *testing.T) { testData := []Backup{ - {metadata.BackupMetadata{BackupName: "three"}, false, "", "", timeParse("2019-03-28T19-50-13")}, - {metadata.BackupMetadata{BackupName: "one"}, false, "", "", timeParse("2019-03-28T19-50-11")}, - {metadata.BackupMetadata{BackupName: "five"}, false, "", "", timeParse("2019-03-28T19-50-15")}, - {metadata.BackupMetadata{BackupName: "two"}, false, "", "", timeParse("2019-03-28T19-50-12")}, - {metadata.BackupMetadata{BackupName: "four"}, false, "", "", timeParse("2019-03-28T19-50-14")}, + {metadata.BackupMetadata{BackupName: "three"}, "", timeParse("2019-03-28T19-50-13")}, + {metadata.BackupMetadata{BackupName: "one"}, "", timeParse("2019-03-28T19-50-11")}, + {metadata.BackupMetadata{BackupName: "five"}, "", timeParse("2019-03-28T19-50-15")}, + {metadata.BackupMetadata{BackupName: "two"}, "", timeParse("2019-03-28T19-50-12")}, + {metadata.BackupMetadata{BackupName: "four"}, "", timeParse("2019-03-28T19-50-14")}, } expectedData := []Backup{ - {metadata.BackupMetadata{BackupName: "two"}, false, "", "", timeParse("2019-03-28T19-50-12")}, - {metadata.BackupMetadata{BackupName: "one"}, false, "", "", timeParse("2019-03-28T19-50-11")}, + {metadata.BackupMetadata{BackupName: "two"}, "", timeParse("2019-03-28T19-50-12")}, + {metadata.BackupMetadata{BackupName: "one"}, "", timeParse("2019-03-28T19-50-11")}, } - assert.Equal(t, expectedData, GetBackupsToDelete(testData, 3)) - assert.Equal(t, []Backup{}, GetBackupsToDelete([]Backup{testData[0]}, 3)) + assert.Equal(t, expectedData, GetBackupsToDeleteRemote(testData, 3)) + assert.Equal(t, []Backup{}, GetBackupsToDeleteRemote([]Backup{testData[0]}, 3)) } func TestGetBackupsToDeleteWithRequiredBackup(t *testing.T) { // fix https://github.com/Altinity/clickhouse-backup/issues/111 testData := []Backup{ - {metadata.BackupMetadata{BackupName: "3"}, false, "", "", timeParse("2019-03-28T19-50-13")}, - {metadata.BackupMetadata{BackupName: "1"}, false, "", "", timeParse("2019-03-28T19-50-11")}, - {metadata.BackupMetadata{BackupName: "5", RequiredBackup: "2"}, false, "", "", timeParse("2019-03-28T19-50-15")}, - {metadata.BackupMetadata{BackupName: "2"}, false, "", "", timeParse("2019-03-28T19-50-12")}, - {metadata.BackupMetadata{BackupName: "4", RequiredBackup: "3"}, false, "", "", timeParse("2019-03-28T19-50-14")}, + {metadata.BackupMetadata{BackupName: "3"}, "", timeParse("2019-03-28T19-50-13")}, + {metadata.BackupMetadata{BackupName: "1"}, "", timeParse("2019-03-28T19-50-11")}, + {metadata.BackupMetadata{BackupName: "5", RequiredBackup: "2"}, "", timeParse("2019-03-28T19-50-15")}, + {metadata.BackupMetadata{BackupName: "2"}, "", timeParse("2019-03-28T19-50-12")}, + {metadata.BackupMetadata{BackupName: "4", RequiredBackup: "3"}, "", timeParse("2019-03-28T19-50-14")}, } expectedData := []Backup{ - {metadata.BackupMetadata{BackupName: "1"}, false, "", "", timeParse("2019-03-28T19-50-11")}, + {metadata.BackupMetadata{BackupName: "1"}, "", timeParse("2019-03-28T19-50-11")}, } - assert.Equal(t, expectedData, GetBackupsToDelete(testData, 3)) - assert.Equal(t, []Backup{}, GetBackupsToDelete([]Backup{testData[0]}, 3)) + assert.Equal(t, expectedData, GetBackupsToDeleteRemote(testData, 3)) + assert.Equal(t, []Backup{}, GetBackupsToDeleteRemote([]Backup{testData[0]}, 3)) // fix https://github.com/Altinity/clickhouse-backup/issues/385 testData = []Backup{ - {metadata.BackupMetadata{BackupName: "3", RequiredBackup: "2"}, false, "", "", timeParse("2019-03-28T19-50-13")}, - {metadata.BackupMetadata{BackupName: "1"}, false, "", "", timeParse("2019-03-28T19-50-11")}, - {metadata.BackupMetadata{BackupName: "5", RequiredBackup: "4"}, false, "", "", timeParse("2019-03-28T19-50-15")}, - {metadata.BackupMetadata{BackupName: "2", RequiredBackup: "1"}, false, "", "", timeParse("2019-03-28T19-50-12")}, - {metadata.BackupMetadata{BackupName: "4", RequiredBackup: "3"}, false, "", "", timeParse("2019-03-28T19-50-14")}, + {metadata.BackupMetadata{BackupName: "3", RequiredBackup: "2"}, "", timeParse("2019-03-28T19-50-13")}, + {metadata.BackupMetadata{BackupName: "1"}, "", timeParse("2019-03-28T19-50-11")}, + {metadata.BackupMetadata{BackupName: "5", RequiredBackup: "4"}, "", timeParse("2019-03-28T19-50-15")}, + {metadata.BackupMetadata{BackupName: "2", RequiredBackup: "1"}, "", timeParse("2019-03-28T19-50-12")}, + {metadata.BackupMetadata{BackupName: "4", RequiredBackup: "3"}, "", timeParse("2019-03-28T19-50-14")}, } expectedData = []Backup{} - assert.Equal(t, expectedData, GetBackupsToDelete(testData, 3)) - assert.Equal(t, []Backup{}, GetBackupsToDelete([]Backup{testData[0]}, 3)) + assert.Equal(t, expectedData, GetBackupsToDeleteRemote(testData, 3)) + assert.Equal(t, []Backup{}, GetBackupsToDeleteRemote([]Backup{testData[0]}, 3)) } func TestGetBackupsToDeleteWithInvalidUploadDate(t *testing.T) { // fix https://github.com/Altinity/clickhouse-backup/issues/409 testData := []Backup{ - {metadata.BackupMetadata{BackupName: "1"}, false, "", "", timeParse("2022-03-03T18-08-01")}, - {metadata.BackupMetadata{BackupName: "2"}, false, "", "", timeParse("2022-03-03T18-08-02")}, - {BackupMetadata: metadata.BackupMetadata{BackupName: "3"}, Legacy: false, FileExtension: "", Broken: ""}, // UploadDate initialized with default value - {metadata.BackupMetadata{BackupName: "4"}, false, "", "", timeParse("2022-03-03T18-08-04")}, + {metadata.BackupMetadata{BackupName: "1"}, "", timeParse("2022-03-03T18-08-01")}, + {metadata.BackupMetadata{BackupName: "2"}, "", timeParse("2022-03-03T18-08-02")}, + {BackupMetadata: metadata.BackupMetadata{BackupName: "3"}, Broken: ""}, // UploadDate initialized with default value + {metadata.BackupMetadata{BackupName: "4"}, "", timeParse("2022-03-03T18-08-04")}, } expectedData := []Backup{ - {metadata.BackupMetadata{BackupName: "1"}, false, "", "", timeParse("2022-03-03T18-08-01")}, + {metadata.BackupMetadata{BackupName: "1"}, "", timeParse("2022-03-03T18-08-01")}, } - assert.Equal(t, expectedData, GetBackupsToDelete(testData, 2)) + assert.Equal(t, expectedData, GetBackupsToDeleteRemote(testData, 2)) } func TestGetBackupsToDeleteWithRecursiveRequiredBackups(t *testing.T) { // fix https://github.com/Altinity/clickhouse-backup/issues/525 testData := []Backup{ - {metadata.BackupMetadata{BackupName: "2022-09-01T05-00-01"}, false, "", "", timeParse("2022-09-01T05-00-01")}, - {metadata.BackupMetadata{BackupName: "2022-09-01T21-00-03", RequiredBackup: "2022-09-01T05-00-01"}, false, "", "", timeParse("2022-09-01T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-02T21-00-02", RequiredBackup: "2022-09-01T21-00-03"}, false, "", "", timeParse("2022-09-02T21-00-02")}, - {metadata.BackupMetadata{BackupName: "2022-09-03T21-00-03", RequiredBackup: "2022-09-02T21-00-02"}, false, "", "", timeParse("2022-09-03T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-04T21-00-03", RequiredBackup: "2022-09-04T21-00-03"}, false, "", "", timeParse("2022-09-04T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-05T21-00-03", RequiredBackup: "2022-09-04T21-00-03"}, false, "", "", timeParse("2022-09-05T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-06T21-00-03", RequiredBackup: "2022-09-05T21-00-03"}, false, "", "", timeParse("2022-09-06T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-07T21-00-03", RequiredBackup: "2022-09-06T21-00-03"}, false, "", "", timeParse("2022-09-07T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-08T21-00-03", RequiredBackup: "2022-09-07T21-00-03"}, false, "", "", timeParse("2022-09-08T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-09T21-00-03", RequiredBackup: "2022-09-08T21-00-03"}, false, "", "", timeParse("2022-09-09T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-10T21-00-03", RequiredBackup: "2022-09-09T21-00-03"}, false, "", "", timeParse("2022-09-10T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-11T21-00-03", RequiredBackup: "2022-09-10T21-00-03"}, false, "", "", timeParse("2022-09-11T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-12T21-00-02", RequiredBackup: "2022-09-11T21-00-03"}, false, "", "", timeParse("2022-09-12T21-00-02")}, - {metadata.BackupMetadata{BackupName: "2022-09-13T21-00-03", RequiredBackup: "2022-09-12T21-00-02"}, false, "", "", timeParse("2022-09-13T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-14T21-00-03", RequiredBackup: "2022-09-13T21-00-03"}, false, "", "", timeParse("2022-09-14T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-10-03T09-30-20"}, false, "", "", timeParse("2022-10-03T09-30-20")}, - {metadata.BackupMetadata{BackupName: "2022-10-03T09-39-37", RequiredBackup: "2022-10-03T09-30-20"}, false, "", "", timeParse("2022-10-03T09-39-37")}, - {metadata.BackupMetadata{BackupName: "2022-10-03T09-40-03", RequiredBackup: "2022-10-03T09-39-37"}, false, "", "", timeParse("2022-10-03T09-40-03")}, - {metadata.BackupMetadata{BackupName: "2022-10-03T09-41-31", RequiredBackup: "2022-10-03T09-40-03"}, false, "", "", timeParse("2022-10-03T09-41-31")}, - {metadata.BackupMetadata{BackupName: "2022-10-03T09-52-12", RequiredBackup: "2022-10-03T09-41-31"}, false, "", "", timeParse("2022-10-03T09-52-12")}, - {metadata.BackupMetadata{BackupName: "2022-10-03T10-11-15", RequiredBackup: "2022-10-03T09-52-12"}, false, "", "", timeParse("2022-10-03T10-11-15")}, - {metadata.BackupMetadata{BackupName: "2022-10-03T10-12-38", RequiredBackup: "2022-10-03T10-11-15"}, false, "", "", timeParse("2022-10-03T10-12-38")}, - {metadata.BackupMetadata{BackupName: "2022-10-03T10-12-57", RequiredBackup: "2022-10-03T10-12-38"}, false, "", "", timeParse("2022-10-03T10-12-57")}, - {metadata.BackupMetadata{BackupName: "2022-10-03T10-13-16", RequiredBackup: "2022-10-03T10-12-57"}, false, "", "", timeParse("2022-10-03T10-13-16")}, - {metadata.BackupMetadata{BackupName: "2022-10-03T10-15-32", RequiredBackup: "2022-10-03T10-13-16"}, false, "", "", timeParse("2022-10-03T10-15-32")}, + {metadata.BackupMetadata{BackupName: "2022-09-01T05-00-01"}, "", timeParse("2022-09-01T05-00-01")}, + {metadata.BackupMetadata{BackupName: "2022-09-01T21-00-03", RequiredBackup: "2022-09-01T05-00-01"}, "", timeParse("2022-09-01T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-02T21-00-02", RequiredBackup: "2022-09-01T21-00-03"}, "", timeParse("2022-09-02T21-00-02")}, + {metadata.BackupMetadata{BackupName: "2022-09-03T21-00-03", RequiredBackup: "2022-09-02T21-00-02"}, "", timeParse("2022-09-03T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-04T21-00-03", RequiredBackup: "2022-09-04T21-00-03"}, "", timeParse("2022-09-04T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-05T21-00-03", RequiredBackup: "2022-09-04T21-00-03"}, "", timeParse("2022-09-05T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-06T21-00-03", RequiredBackup: "2022-09-05T21-00-03"}, "", timeParse("2022-09-06T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-07T21-00-03", RequiredBackup: "2022-09-06T21-00-03"}, "", timeParse("2022-09-07T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-08T21-00-03", RequiredBackup: "2022-09-07T21-00-03"}, "", timeParse("2022-09-08T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-09T21-00-03", RequiredBackup: "2022-09-08T21-00-03"}, "", timeParse("2022-09-09T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-10T21-00-03", RequiredBackup: "2022-09-09T21-00-03"}, "", timeParse("2022-09-10T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-11T21-00-03", RequiredBackup: "2022-09-10T21-00-03"}, "", timeParse("2022-09-11T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-12T21-00-02", RequiredBackup: "2022-09-11T21-00-03"}, "", timeParse("2022-09-12T21-00-02")}, + {metadata.BackupMetadata{BackupName: "2022-09-13T21-00-03", RequiredBackup: "2022-09-12T21-00-02"}, "", timeParse("2022-09-13T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-14T21-00-03", RequiredBackup: "2022-09-13T21-00-03"}, "", timeParse("2022-09-14T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-10-03T09-30-20"}, "", timeParse("2022-10-03T09-30-20")}, + {metadata.BackupMetadata{BackupName: "2022-10-03T09-39-37", RequiredBackup: "2022-10-03T09-30-20"}, "", timeParse("2022-10-03T09-39-37")}, + {metadata.BackupMetadata{BackupName: "2022-10-03T09-40-03", RequiredBackup: "2022-10-03T09-39-37"}, "", timeParse("2022-10-03T09-40-03")}, + {metadata.BackupMetadata{BackupName: "2022-10-03T09-41-31", RequiredBackup: "2022-10-03T09-40-03"}, "", timeParse("2022-10-03T09-41-31")}, + {metadata.BackupMetadata{BackupName: "2022-10-03T09-52-12", RequiredBackup: "2022-10-03T09-41-31"}, "", timeParse("2022-10-03T09-52-12")}, + {metadata.BackupMetadata{BackupName: "2022-10-03T10-11-15", RequiredBackup: "2022-10-03T09-52-12"}, "", timeParse("2022-10-03T10-11-15")}, + {metadata.BackupMetadata{BackupName: "2022-10-03T10-12-38", RequiredBackup: "2022-10-03T10-11-15"}, "", timeParse("2022-10-03T10-12-38")}, + {metadata.BackupMetadata{BackupName: "2022-10-03T10-12-57", RequiredBackup: "2022-10-03T10-12-38"}, "", timeParse("2022-10-03T10-12-57")}, + {metadata.BackupMetadata{BackupName: "2022-10-03T10-13-16", RequiredBackup: "2022-10-03T10-12-57"}, "", timeParse("2022-10-03T10-13-16")}, + {metadata.BackupMetadata{BackupName: "2022-10-03T10-15-32", RequiredBackup: "2022-10-03T10-13-16"}, "", timeParse("2022-10-03T10-15-32")}, } expectedData := []Backup{ - {metadata.BackupMetadata{BackupName: "2022-09-14T21-00-03", RequiredBackup: "2022-09-13T21-00-03"}, false, "", "", timeParse("2022-09-14T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-13T21-00-03", RequiredBackup: "2022-09-12T21-00-02"}, false, "", "", timeParse("2022-09-13T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-12T21-00-02", RequiredBackup: "2022-09-11T21-00-03"}, false, "", "", timeParse("2022-09-12T21-00-02")}, - {metadata.BackupMetadata{BackupName: "2022-09-11T21-00-03", RequiredBackup: "2022-09-10T21-00-03"}, false, "", "", timeParse("2022-09-11T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-10T21-00-03", RequiredBackup: "2022-09-09T21-00-03"}, false, "", "", timeParse("2022-09-10T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-09T21-00-03", RequiredBackup: "2022-09-08T21-00-03"}, false, "", "", timeParse("2022-09-09T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-08T21-00-03", RequiredBackup: "2022-09-07T21-00-03"}, false, "", "", timeParse("2022-09-08T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-07T21-00-03", RequiredBackup: "2022-09-06T21-00-03"}, false, "", "", timeParse("2022-09-07T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-06T21-00-03", RequiredBackup: "2022-09-05T21-00-03"}, false, "", "", timeParse("2022-09-06T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-05T21-00-03", RequiredBackup: "2022-09-04T21-00-03"}, false, "", "", timeParse("2022-09-05T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-04T21-00-03", RequiredBackup: "2022-09-04T21-00-03"}, false, "", "", timeParse("2022-09-04T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-03T21-00-03", RequiredBackup: "2022-09-02T21-00-02"}, false, "", "", timeParse("2022-09-03T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-02T21-00-02", RequiredBackup: "2022-09-01T21-00-03"}, false, "", "", timeParse("2022-09-02T21-00-02")}, - {metadata.BackupMetadata{BackupName: "2022-09-01T21-00-03", RequiredBackup: "2022-09-01T05-00-01"}, false, "", "", timeParse("2022-09-01T21-00-03")}, - {metadata.BackupMetadata{BackupName: "2022-09-01T05-00-01"}, false, "", "", timeParse("2022-09-01T05-00-01")}, + {metadata.BackupMetadata{BackupName: "2022-09-14T21-00-03", RequiredBackup: "2022-09-13T21-00-03"}, "", timeParse("2022-09-14T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-13T21-00-03", RequiredBackup: "2022-09-12T21-00-02"}, "", timeParse("2022-09-13T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-12T21-00-02", RequiredBackup: "2022-09-11T21-00-03"}, "", timeParse("2022-09-12T21-00-02")}, + {metadata.BackupMetadata{BackupName: "2022-09-11T21-00-03", RequiredBackup: "2022-09-10T21-00-03"}, "", timeParse("2022-09-11T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-10T21-00-03", RequiredBackup: "2022-09-09T21-00-03"}, "", timeParse("2022-09-10T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-09T21-00-03", RequiredBackup: "2022-09-08T21-00-03"}, "", timeParse("2022-09-09T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-08T21-00-03", RequiredBackup: "2022-09-07T21-00-03"}, "", timeParse("2022-09-08T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-07T21-00-03", RequiredBackup: "2022-09-06T21-00-03"}, "", timeParse("2022-09-07T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-06T21-00-03", RequiredBackup: "2022-09-05T21-00-03"}, "", timeParse("2022-09-06T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-05T21-00-03", RequiredBackup: "2022-09-04T21-00-03"}, "", timeParse("2022-09-05T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-04T21-00-03", RequiredBackup: "2022-09-04T21-00-03"}, "", timeParse("2022-09-04T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-03T21-00-03", RequiredBackup: "2022-09-02T21-00-02"}, "", timeParse("2022-09-03T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-02T21-00-02", RequiredBackup: "2022-09-01T21-00-03"}, "", timeParse("2022-09-02T21-00-02")}, + {metadata.BackupMetadata{BackupName: "2022-09-01T21-00-03", RequiredBackup: "2022-09-01T05-00-01"}, "", timeParse("2022-09-01T21-00-03")}, + {metadata.BackupMetadata{BackupName: "2022-09-01T05-00-01"}, "", timeParse("2022-09-01T05-00-01")}, } - assert.Equal(t, expectedData, GetBackupsToDelete(testData, 6)) + assert.Equal(t, expectedData, GetBackupsToDeleteRemote(testData, 6)) } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 9720a402..9e09afcd 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -61,13 +61,13 @@ func HumanizeDuration(d time.Duration) string { func ExecCmd(ctx context.Context, timeout time.Duration, cmd string, args ...string) error { out, err := ExecCmdOut(ctx, timeout, cmd, args...) - log.Info().Msg(out) + log.Debug().Msg(out) return err } func ExecCmdOut(ctx context.Context, timeout time.Duration, cmd string, args ...string) (string, error) { ctx, cancel := context.WithTimeout(ctx, timeout) - log.Info().Msgf("%s %s", cmd, strings.Join(args, " ")) + log.Debug().Msgf("%s %s", cmd, strings.Join(args, " ")) out, err := exec.CommandContext(ctx, cmd, args...).CombinedOutput() cancel() return string(out), err diff --git a/test/integration/cluster.xml b/test/integration/clickhouse-config.xml similarity index 100% rename from test/integration/cluster.xml rename to test/integration/clickhouse-config.xml diff --git a/test/integration/config-azblob-embedded-url.yml b/test/integration/config-azblob-embedded-url.yml new file mode 100644 index 00000000..2a2cd4b8 --- /dev/null +++ b/test/integration/config-azblob-embedded-url.yml @@ -0,0 +1,37 @@ +general: + disable_progress_bar: true + remote_storage: azblob + upload_concurrency: 4 + download_concurrency: 4 + skip_tables: + - " system.*" + - "INFORMATION_SCHEMA.*" + - "information_schema.*" + - "_temporary_and_external_tables.*" + restore_schema_on_cluster: "{cluster}" +clickhouse: + host: clickhouse + port: 9440 + username: backup + password: meow=& 123?*%# МЯУ + secure: true + skip_verify: true + sync_replicated_tables: true + timeout: 4h + restart_command: bash -c 'echo "FAKE RESTART"' + use_embedded_backup_restore: true + embedded_backup_disk: "" +azblob: + account_name: devstoreaccount1 + account_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + endpoint_suffix: azure:10000 + endpoint_schema: http + container: container1 + object_disk_path: object_disk/{cluster}/{shard} + path: backup + compression_format: none +api: + listen: :7171 + create_integration_tables: true + integration_tables_host: "clickhouse-backup" + allow_parallel: true diff --git a/test/integration/config-azblob.yml b/test/integration/config-azblob.yml index 29ef5be1..8477246f 100644 --- a/test/integration/config-azblob.yml +++ b/test/integration/config-azblob.yml @@ -8,6 +8,7 @@ clickhouse: host: clickhouse port: 9000 restart_command: bash -c 'echo "FAKE RESTART"' + timeout: 60s azblob: account_name: devstoreaccount1 account_key: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== diff --git a/test/integration/config-custom-kopia.yml b/test/integration/config-custom-kopia.yml index 26b70025..8fb991db 100644 --- a/test/integration/config-custom-kopia.yml +++ b/test/integration/config-custom-kopia.yml @@ -15,10 +15,10 @@ clickhouse: username: backup password: meow=& 123?*%# МЯУ sync_replicated_tables: true - timeout: 5s restart_command: "sql:SYSTEM RELOAD USERS; sql:SYSTEM RELOAD CONFIG; sql:SYSTEM SHUTDOWN" + timeout: 60s custom: - # all `kopia` uploads are incremental + # all `kopia` uploads are incremental we don't need {{ .diffFromRemote }} upload_command: /custom/kopia/upload.sh {{ .backupName }} download_command: /custom/kopia/download.sh {{ .backupName }} delete_command: /custom/kopia/delete.sh {{ .backupName }} diff --git a/test/integration/config-custom-restic.yml b/test/integration/config-custom-restic.yml index 88372f07..69a9bf48 100644 --- a/test/integration/config-custom-restic.yml +++ b/test/integration/config-custom-restic.yml @@ -15,8 +15,8 @@ clickhouse: username: backup password: meow=& 123?*%# МЯУ sync_replicated_tables: true - timeout: 5s restart_command: "sql:SYSTEM RELOAD USERS; sql:SYSTEM RELOAD CONFIG; sql:SYSTEM SHUTDOWN" + timeout: 60s custom: upload_command: /custom/restic/upload.sh {{ .backupName }} {{ .diffFromRemote }} download_command: /custom/restic/download.sh {{ .backupName }} diff --git a/test/integration/config-custom-rsync.yml b/test/integration/config-custom-rsync.yml index 74965d84..b8671975 100644 --- a/test/integration/config-custom-rsync.yml +++ b/test/integration/config-custom-rsync.yml @@ -15,8 +15,8 @@ clickhouse: username: backup password: meow=& 123?*%# МЯУ sync_replicated_tables: true - timeout: 5s restart_command: "sql:SYSTEM RELOAD USERS; sql:SYSTEM RELOAD CONFIG; sql:SYSTEM SHUTDOWN" + timeout: 60s custom: upload_command: /custom/rsync/upload.sh {{ .backupName }} {{ .diffFromRemote }} download_command: /custom/rsync/download.sh {{ .backupName }} diff --git a/test/integration/config-database-mapping.yml b/test/integration/config-database-mapping.yml index d73b2ee9..d6b6b489 100644 --- a/test/integration/config-database-mapping.yml +++ b/test/integration/config-database-mapping.yml @@ -14,18 +14,19 @@ clickhouse: secure: true skip_verify: true sync_replicated_tables: true - timeout: 1s restart_command: bash -c 'echo "FAKE RESTART"' + timeout: 60s s3: - access_key: access-key - secret_key: it-is-my-super-secret-key + access_key: access_key + secret_key: it_is_my_super_secret_key bucket: clickhouse - endpoint: http://minio:9000 + endpoint: https://minio:9000 + disable_ssl: false + disable_cert_verification: true acl: private force_path_style: true path: backup/{cluster}/{shard} object_disk_path: object_disk/{cluster}/{shard} - disable_ssl: true compression_format: tar api: listen: :7171 diff --git a/test/integration/config-ftp-old.yaml b/test/integration/config-ftp-old.yaml new file mode 100644 index 00000000..083b3314 --- /dev/null +++ b/test/integration/config-ftp-old.yaml @@ -0,0 +1,27 @@ +general: + disable_progress_bar: true + remote_storage: ftp + upload_concurrency: 4 + download_concurrency: 4 + restore_schema_on_cluster: "{cluster}" +clickhouse: + restore_as_attach: false + host: clickhouse + port: 9440 + username: backup + password: meow=& 123?*%# МЯУ + secure: true + skip_verify: true + restart_command: bash -c 'echo "FAKE RESTART"' + timeout: 60s +ftp: + address: "ftp:21" + username: "test_backup" + password: "test_backup" + tls: false + path: "/backup" + compression_format: none + compression_level: 1 + concurrency: 4 +api: + listen: :7171 diff --git a/test/integration/config-ftp.yaml b/test/integration/config-ftp.yaml index 0f0b6d8a..8c0da918 100644 --- a/test/integration/config-ftp.yaml +++ b/test/integration/config-ftp.yaml @@ -2,9 +2,12 @@ general: disable_progress_bar: true remote_storage: ftp upload_concurrency: 4 + upload_max_bytes_per_second: 10240 + download_max_bytes_per_second: 10240 download_concurrency: 4 restore_schema_on_cluster: "{cluster}" clickhouse: + restore_as_attach: true host: clickhouse port: 9440 username: backup @@ -12,6 +15,7 @@ clickhouse: secure: true skip_verify: true restart_command: bash -c 'echo "FAKE RESTART"' + timeout: 60s ftp: address: "ftp:21" username: "test_backup" diff --git a/test/integration/config-gcs-custom-endpoint.yml b/test/integration/config-gcs-custom-endpoint.yml new file mode 100644 index 00000000..f1f354ae --- /dev/null +++ b/test/integration/config-gcs-custom-endpoint.yml @@ -0,0 +1,30 @@ +general: + disable_progress_bar: true + remote_storage: gcs + upload_concurrency: 4 + download_concurrency: 4 + skip_tables: + - " system.*" + - "INFORMATION_SCHEMA.*" + - "information_schema.*" + - "_temporary_and_external_tables.*" + restore_schema_on_cluster: "{cluster}" +clickhouse: + host: clickhouse + port: 9440 + username: backup + password: meow=& 123?*%# МЯУ + secure: true + skip_verify: true + sync_replicated_tables: true + restart_command: "sql:SYSTEM RELOAD USERS; sql:SYSTEM RELOAD CONFIG; exec:ls -la /var/lib/clickhouse/access; sql:SYSTEM SHUTDOWN" + # restart_command: bash -c 'echo "FAKE RESTART"' + backup_mutations: true + timeout: 60s +gcs: + bucket: altinity-qa-test + path: backup/{cluster}/{shard} + object_disk_path: object_disks/{cluster}/{shard} + compression_format: tar + endpoint: http://gcs:8080/storage/v1/ + skip_credentials: true diff --git a/test/integration/config-gcs-embedded-url.yml b/test/integration/config-gcs-embedded-url.yml new file mode 100644 index 00000000..6921c1da --- /dev/null +++ b/test/integration/config-gcs-embedded-url.yml @@ -0,0 +1,20 @@ +general: + disable_progress_bar: true + remote_storage: gcs + upload_concurrency: 4 + download_concurrency: 4 + restore_schema_on_cluster: "{cluster}" +clickhouse: + host: clickhouse + port: 9000 + restart_command: bash -c 'echo "FAKE RESTART"' + use_embedded_backup_restore: true + timeout: 4h +gcs: + bucket: "${QA_GCS_OVER_S3_BUCKET}" + path: backup/{cluster}/{shard} + object_disk_path: object_disks/{cluster}/{shard} + credentials_file: /etc/clickhouse-backup/credentials.json + embedded_access_key: "${QA_GCS_OVER_S3_ACCESS_KEY}" + embedded_secret_key: "${QA_GCS_OVER_S3_SECRET_KEY}" + compression_format: none diff --git a/test/integration/config-gcs.yml b/test/integration/config-gcs.yml index 55996304..514de3a4 100644 --- a/test/integration/config-gcs.yml +++ b/test/integration/config-gcs.yml @@ -8,10 +8,10 @@ clickhouse: host: clickhouse port: 9000 restart_command: bash -c 'echo "FAKE RESTART"' + timeout: 60s gcs: bucket: altinity-qa-test - path: backup - object_disk_path: object_disks + path: backup/{cluster}/{shard} + object_disk_path: object_disks/{cluster}/{shard} credentials_file: /etc/clickhouse-backup/credentials.json compression_format: tar -# endpoint: http://gcs:8080/storage/v1/ diff --git a/test/integration/config-s3-embedded-local.yml b/test/integration/config-s3-embedded-local.yml new file mode 100644 index 00000000..6b3f1f1f --- /dev/null +++ b/test/integration/config-s3-embedded-local.yml @@ -0,0 +1,40 @@ +general: + disable_progress_bar: true + remote_storage: s3 + upload_concurrency: 4 + download_concurrency: 4 + skip_tables: + - " system.*" + - "INFORMATION_SCHEMA.*" + - "information_schema.*" + - "_temporary_and_external_tables.*" + restore_schema_on_cluster: "{cluster}" +clickhouse: + host: clickhouse + port: 9440 + username: backup + password: meow=& 123?*%# МЯУ + secure: true + skip_verify: true + sync_replicated_tables: true + timeout: 4h + restart_command: bash -c 'echo "FAKE RESTART"' + use_embedded_backup_restore: true + embedded_backup_disk: backups_local +s3: + access_key: access_key + secret_key: it_is_my_super_secret_key + bucket: clickhouse + endpoint: https://minio:9000 + disable_ssl: false + disable_cert_verification: true + acl: private + force_path_style: true + path: backup/{cluster}/{shard} + object_disk_path: object_disk/{cluster}/{shard} + compression_format: tar +api: + listen: :7171 + create_integration_tables: true + integration_tables_host: "clickhouse-backup" + allow_parallel: true diff --git a/test/integration/config-s3-embedded-url.yml b/test/integration/config-s3-embedded-url.yml new file mode 100644 index 00000000..ec392980 --- /dev/null +++ b/test/integration/config-s3-embedded-url.yml @@ -0,0 +1,40 @@ +general: + disable_progress_bar: true + remote_storage: s3 + upload_concurrency: 4 + download_concurrency: 4 + skip_tables: + - " system.*" + - "INFORMATION_SCHEMA.*" + - "information_schema.*" + - "_temporary_and_external_tables.*" + restore_schema_on_cluster: "{cluster}" +clickhouse: + host: clickhouse + port: 9440 + username: backup + password: meow=& 123?*%# МЯУ + secure: true + skip_verify: true + sync_replicated_tables: true + timeout: 4h + restart_command: bash -c 'echo "FAKE RESTART"' + use_embedded_backup_restore: true + embedded_backup_disk: "" +s3: + access_key: access_key + secret_key: it_is_my_super_secret_key + bucket: clickhouse + endpoint: https://minio:9000 + disable_ssl: false + disable_cert_verification: true + acl: private + force_path_style: true + path: backup/{cluster}/{shard} + object_disk_path: object_disk/{cluster}/{shard} + compression_format: tar +api: + listen: :7171 + create_integration_tables: true + integration_tables_host: "clickhouse-backup" + allow_parallel: true diff --git a/test/integration/config-s3-embedded.yml b/test/integration/config-s3-embedded.yml index d8071dcd..cf2b6188 100644 --- a/test/integration/config-s3-embedded.yml +++ b/test/integration/config-s3-embedded.yml @@ -22,15 +22,17 @@ clickhouse: use_embedded_backup_restore: true embedded_backup_disk: backups_s3 s3: - access_key: access-key - secret_key: it-is-my-super-secret-key + access_key: access_key + secret_key: it_is_my_super_secret_key bucket: clickhouse - endpoint: http://minio:9000 + endpoint: https://minio:9000 + disable_ssl: false + disable_cert_verification: true acl: private force_path_style: true path: backup/{cluster}/{shard} - disable_ssl: true - compression_format: none + object_disk_path: object_disk/{cluster}/{shard} + compression_format: tar api: listen: :7171 create_integration_tables: true diff --git a/test/integration/config-s3-fips.yml b/test/integration/config-s3-fips.yml index 7fe65ff0..f856377b 100644 --- a/test/integration/config-s3-fips.yml +++ b/test/integration/config-s3-fips.yml @@ -17,9 +17,9 @@ clickhouse: secure: true skip_verify: true sync_replicated_tables: true - timeout: 2s restart_command: bash -c 'echo "FAKE RESTART"' backup_mutations: true + timeout: 60s # secrets for `FISP` will provide from `.env` or from GitHub actions secrets s3: access_key: ${QA_AWS_ACCESS_KEY} diff --git a/test/integration/config-s3-nodelete.yml b/test/integration/config-s3-nodelete.yml index b5e093be..cc7ec361 100644 --- a/test/integration/config-s3-nodelete.yml +++ b/test/integration/config-s3-nodelete.yml @@ -17,18 +17,19 @@ clickhouse: secure: true skip_verify: true sync_replicated_tables: true - timeout: 1s restart_command: bash -c 'echo "FAKE RESTART"' + timeout: 60s s3: access_key: nodelete secret_key: nodelete_password bucket: clickhouse - endpoint: http://minio:9000 + endpoint: https://minio:9000 + disable_ssl: false + disable_cert_verification: true acl: private force_path_style: true path: backup/{cluster}/{shard} object_disk_path: object_disk/{cluster}/{shard} - disable_ssl: true compression_format: tar api: listen: :7171 diff --git a/test/integration/config-s3-plain-embedded.yml b/test/integration/config-s3-plain-embedded.yml index 3060aea1..2ec7f302 100644 --- a/test/integration/config-s3-plain-embedded.yml +++ b/test/integration/config-s3-plain-embedded.yml @@ -22,15 +22,16 @@ clickhouse: use_embedded_backup_restore: true embedded_backup_disk: backups_s3_plain s3: - access_key: access-key - secret_key: it-is-my-super-secret-key + access_key: access_key + secret_key: it_is_my_super_secret_key bucket: clickhouse - endpoint: http://minio:9000 + endpoint: https://minio:9000 + disable_ssl: false + disable_cert_verification: true acl: private force_path_style: true path: backup/{cluster}/{shard} object_disk_path: object_disk/{cluster}/{shard} - disable_ssl: true compression_format: none api: listen: :7171 diff --git a/test/integration/config-s3.yml b/test/integration/config-s3.yml index 8e20fbd3..8138afa3 100644 --- a/test/integration/config-s3.yml +++ b/test/integration/config-s3.yml @@ -20,22 +20,25 @@ clickhouse: secure: true skip_verify: true sync_replicated_tables: true - timeout: 5s - restart_command: "sql:SYSTEM RELOAD USERS; sql:SYSTEM RELOAD CONFIG; sql:SYSTEM SHUTDOWN" + restart_command: "sql:SYSTEM RELOAD USERS; sql:SYSTEM RELOAD CONFIG; exec:ls -la /var/lib/clickhouse/access; sql:SYSTEM SHUTDOWN" + # restart_command: bash -c 'echo "FAKE RESTART"' backup_mutations: true + timeout: 60s s3: - access_key: access-key - secret_key: it-is-my-super-secret-key + access_key: access_key + secret_key: it_is_my_super_secret_key bucket: clickhouse - endpoint: http://minio:9000 + endpoint: https://minio:9000 + disable_ssl: false + disable_cert_verification: true acl: private force_path_style: true path: backup/{cluster}/{shard} object_disk_path: object_disk/{cluster}/{shard} - disable_ssl: true compression_format: tar allow_multipart_download: true concurrency: 3 + request_payer: requester api: listen: :7171 create_integration_tables: true diff --git a/test/integration/config-sftp-auth-key.yaml b/test/integration/config-sftp-auth-key.yaml index d7037c85..89efaaf2 100644 --- a/test/integration/config-sftp-auth-key.yaml +++ b/test/integration/config-sftp-auth-key.yaml @@ -11,6 +11,7 @@ clickhouse: secure: true skip_verify: true restart_command: bash -c 'echo "FAKE RESTART"' + timeout: 60s sftp: address: "sshd" username: "root" diff --git a/test/integration/config-sftp-auth-password.yaml b/test/integration/config-sftp-auth-password.yaml index 55191d5f..e862b4a0 100644 --- a/test/integration/config-sftp-auth-password.yaml +++ b/test/integration/config-sftp-auth-password.yaml @@ -12,6 +12,7 @@ clickhouse: secure: true skip_verify: true restart_command: bash -c 'echo "FAKE RESTART"' + timeout: 60s sftp: address: "sshd" username: "root" diff --git a/test/integration/custom_entrypoint.sh b/test/integration/custom_entrypoint.sh new file mode 100755 index 00000000..3a0f5c7f --- /dev/null +++ b/test/integration/custom_entrypoint.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# to avoid backward incompatibility ;( +# https://t.me/clickhouse_ru/359960 +# https://t.me/clickhouse_ru/359968 +# https://t.me/clickhouse_ru/362378 + +if [ $# -ne 0 ]; then + /entrypoint.sh "$@" +else + /docker-entrypoint-initdb.d/dynamic_settings.sh + /entrypoint.sh +fi diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml index a5f9d012..4eb60542 100644 --- a/test/integration/docker-compose.yml +++ b/test/integration/docker-compose.yml @@ -1,13 +1,11 @@ services: sshd: image: docker.io/panubo/sshd:latest - container_name: sshd + hostname: sshd environment: SSH_ENABLE_ROOT: "true" SSH_ENABLE_PASSWORD_AUTH: "true" command: sh -c 'echo "PermitRootLogin yes" >> /etc/ssh/sshd_config && echo "LogLevel DEBUG3" >> /etc/ssh/sshd_config && echo "root:JFzMHfVpvTgEd74XXPq6wARA2Qg3AutJ" | chpasswd && /usr/sbin/sshd -D -e -f /etc/ssh/sshd_config' - networks: - - clickhouse-backup ftp: image: docker.io/fauria/vsftpd:latest @@ -20,51 +18,58 @@ services: PASV_ADDR_RESOLVE: "YES" PASV_MIN_PORT: 20000 PASV_MAX_PORT: 21000 - networks: - - clickhouse-backup minio: image: docker.io/bitnami/minio:${MINIO_VERSION:-latest} - container_name: minio + hostname: minio environment: - MINIO_ACCESS_KEY: access-key - MINIO_SECRET_KEY: it-is-my-super-secret-key MINIO_DEFAULT_BUCKETS: 'clickhouse' - MINIO_ROOT_USER: access-key - MINIO_ROOT_PASSWORD: it-is-my-super-secret-key + MINIO_ROOT_USER: access_key + MINIO_ROOT_PASSWORD: it_is_my_super_secret_key + MINIO_SCHEME: https + BITNAMI_DEBUG: "true" healthcheck: - test: curl -sL http://localhost:9000/ - interval: 10s - retries: 30 + test: ls -la /bitnami/minio/data/clickhouse/ && curl -skL https://localhost:9000/ + interval: 1s + retries: 60 volumes: - ./minio_nodelete.sh:/bin/minio_nodelete.sh - networks: - - clickhouse-backup - -# todo need to reproduce download after upload -# gcs: -# image: fsouza/fake-gcs-server:latest -# hostname: gcs -# entrypoint: -# - /bin/sh -# command: -# - -c -# - "mkdir -p /data/clickhouse-backup-test-gcs && fake-gcs-server -data /data -scheme http -port 8080 -public-host gsc:8080" -# networks: -# - clickhouse-backup + # go install github.com/minio/certgen@latest + # ~/go/bin/certgen -host "127.0.0.1,localhost,minio" + - ./minio.crt:/certs/CAs/public.crt + - ./minio.crt:/certs/public.crt + - ./minio.key:/certs/private.key + gcs: + image: fsouza/fake-gcs-server:latest + hostname: gcs + entrypoint: + - /bin/sh + command: + - -c + - "mkdir -p /data/altinity-qa-test && mkdir -p /data/${QA_GCS_OVER_S3_BUCKET} && fake-gcs-server -data /data -scheme http -port 8080 -public-host gcs:8080" + environment: + QA_GCS_OVER_S3_BUCKET: "${QA_GCS_OVER_S3_BUCKET}" + healthcheck: + test: nc 127.0.0.1 8080 -z + interval: 1s + retries: 30 azure: + # todo need to blobKeepAliveTimeout, and merge https://github.com/Azure/Azurite/pull/2443 image: mcr.microsoft.com/azure-storage/azurite:latest - container_name: azure + # image: docker.io/clickhousepro/azurite:latest hostname: devstoreaccount1.blob.azure healthcheck: test: nc 127.0.0.1 10000 -z interval: 1s retries: 30 - command: ["azurite", "--debug", "/data/debug.log" , "-l", "/data", "--blobHost", "0.0.0.0","--queueHost", "0.0.0.0", "--tableHost", "0.0.0.0"] - networks: - - clickhouse-backup + command: [ "azurite", "--debug", "/data/debug.log" , "-l", "/data", "--blobHost", "0.0.0.0" ] + # command: [ "azurite", "--debug", "/data/debug.log" , "-l", "/data", "--blobHost", "0.0.0.0","--blobKeepAliveTimeout", "3600" ] + volumes: + - azure:/data +# environment: +# - AZURITE_DB="mysql://root:root@mysql:3306/azurite_blob" # azure_init: # image: mcr.microsoft.com/azure-cli:latest @@ -80,28 +85,24 @@ services: # environment: # # https://github.com/Azure/Azurite/blob/main/README.md#usage-with-azure-storage-sdks-or-tools # AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure:10000/devstoreaccount1; - # networks: - # - clickhouse-backup zookeeper: - image: docker.io/zookeeper:${ZOOKEEPER_VERSION:-latest} + # @TODO back :latest default value after resolve https://github.com/ClickHouse/ClickHouse/issues/53749 + image: ${ZOOKEEPER_IMAGE:-docker.io/zookeeper}:${ZOOKEEPER_VERSION:-3.8.4} hostname: zookeeper environment: ZOO_4LW_COMMANDS_WHITELIST: "*" - networks: - - clickhouse-backup healthcheck: test: bash -c 'if [[ "$$(echo 'ruok' | nc 127.0.0.1 2181)" == "imok" ]]; then exit 0; else exit 1; fi' - interval: 3s + interval: 1s timeout: 2s - retries: 5 - start_period: 2s + retries: 10 + start_period: 1s clickhouse-backup: image: docker.io/${CLICKHOUSE_IMAGE:-yandex/clickhouse-server}:${CLICKHOUSE_VERSION:-1.1.54390} hostname: clickhouse-backup - container_name: clickhouse-backup user: root entrypoint: - /bin/bash @@ -109,7 +110,7 @@ services: - sleep infinity healthcheck: test: bash -c "exit 0" - interval: 30s + interval: 1s timeout: 1s retries: 5 start_period: 1s @@ -121,27 +122,27 @@ services: GCS_DEBUG: "${GCS_DEBUG:-false}" FTP_DEBUG: "${FTP_DEBUG:-false}" SFTP_DEBUG: "${SFTP_DEBUG:-false}" + AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}" CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}" GOCOVERDIR: "/tmp/_coverage_/" -# fake-gcs-server -# STORAGE_EMULATOR_HOST: "http://gsc:8080" -# GOOGLE_API_USE_CLIENT_CERTIFICATE: "false" # FIPS QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY} QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY} QA_AWS_BUCKET: ${QA_AWS_BUCKET} QA_AWS_REGION: ${QA_AWS_REGION} +# GCS over S3 embedded backups + QA_GCS_OVER_S3_ACCESS_KEY: "${QA_GCS_OVER_S3_ACCESS_KEY}" + QA_GCS_OVER_S3_SECRET_KEY: "${QA_GCS_OVER_S3_SECRET_KEY}" + QA_GCS_OVER_S3_BUCKET: "${QA_GCS_OVER_S3_BUCKET}" # https://github.com/Altinity/clickhouse-backup/issues/691: - AWS_ACCESS_KEY_ID: access-key - AWS_SECRET_ACCESS_KEY: it-is-my-super-secret-key + AWS_ACCESS_KEY_ID: access_key + AWS_SECRET_ACCESS_KEY: it_is_my_super_secret_key volumes_from: - clickhouse ports: - - "7171:7171" + - "7171" # for delve debugger -# - "40001:40001" - networks: - - clickhouse-backup +# - "40001" depends_on: clickhouse: condition: service_healthy @@ -149,30 +150,28 @@ services: clickhouse: image: docker.io/${CLICKHOUSE_IMAGE:-yandex/clickhouse-server}:${CLICKHOUSE_VERSION:-1.1.54390} hostname: clickhouse - container_name: clickhouse restart: always user: root environment: CLICKHOUSE_VERSION: ${CLICKHOUSE_VERSION:-1.1.54394} + CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS: "true" TZ: UTC LOG_LEVEL: "${LOG_LEVEL:-info}" S3_DEBUG: "${S3_DEBUG:-false}" GCS_DEBUG: "${GCS_DEBUG:-false}" FTP_DEBUG: "${FTP_DEBUG:-false}" SFTP_DEBUG: "${SFTP_DEBUG:-false}" + AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}" CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}" GOCOVERDIR: "/tmp/_coverage_/" -# fake-gcs-server -# STORAGE_EMULATOR_HOST: "http://gsc:8080" -# GOOGLE_API_USE_CLIENT_CERTIFICATE: "false" # FIPS QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY} QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY} QA_AWS_BUCKET: ${QA_AWS_BUCKET} QA_AWS_REGION: ${QA_AWS_REGION} # https://github.com/Altinity/clickhouse-backup/issues/691: - AWS_ACCESS_KEY_ID: access-key - AWS_SECRET_ACCESS_KEY: it-is-my-super-secret-key + AWS_ACCESS_KEY_ID: access_key + AWS_SECRET_ACCESS_KEY: it_is_my_super_secret_key # GCS over S3 object disk QA_GCS_OVER_S3_ACCESS_KEY: "${QA_GCS_OVER_S3_ACCESS_KEY}" QA_GCS_OVER_S3_SECRET_KEY: "${QA_GCS_OVER_S3_SECRET_KEY}" @@ -185,14 +184,20 @@ services: - ./credentials.json:/etc/clickhouse-backup/credentials.json - ./config-azblob.yml:/etc/clickhouse-backup/config-azblob.yml - ./config-azblob-embedded.yml:/etc/clickhouse-backup/config-azblob-embedded.yml + - ./config-azblob-embedded-url.yml:/etc/clickhouse-backup/config-azblob-embedded-url.yml - ./config-custom-kopia.yml:/etc/clickhouse-backup/config-custom-kopia.yml - ./config-custom-restic.yml:/etc/clickhouse-backup/config-custom-restic.yml - ./config-custom-rsync.yml:/etc/clickhouse-backup/config-custom-rsync.yml - ./config-database-mapping.yml:/etc/clickhouse-backup/config-database-mapping.yml - ./config-ftp.yaml:/etc/clickhouse-backup/config-ftp.yaml + - ./config-ftp-old.yaml:/etc/clickhouse-backup/config-ftp-old.yaml - ./config-gcs.yml:/etc/clickhouse-backup/config-gcs.yml + - ./config-gcs-embedded-url.yml:/etc/clickhouse-backup/config-gcs-embedded-url.yml.template + - ./config-gcs-custom-endpoint.yml:/etc/clickhouse-backup/config-gcs-custom-endpoint.yml - ./config-s3.yml:/etc/clickhouse-backup/config-s3.yml - ./config-s3-embedded.yml:/etc/clickhouse-backup/config-s3-embedded.yml + - ./config-s3-embedded-url.yml:/etc/clickhouse-backup/config-s3-embedded-url.yml + - ./config-s3-embedded-local.yml:/etc/clickhouse-backup/config-s3-embedded-local.yml - ./config-s3-fips.yml:/etc/clickhouse-backup/config-s3-fips.yml.template - ./config-s3-nodelete.yml:/etc/clickhouse-backup/config-s3-nodelete.yml - ./config-s3-plain-embedded.yml:/etc/clickhouse-backup/config-s3-plain-embedded.yml @@ -207,7 +212,7 @@ services: - ./server.key:/etc/clickhouse-server/server.key - ./dhparam.pem:/etc/clickhouse-server/dhparam.pem - ./ssl.xml:/etc/clickhouse-server/config.d/ssl.xml - - ./cluster.xml:/etc/clickhouse-server/config.d/cluster.xml + - ./clickhouse-config.xml:/etc/clickhouse-server/config.d/clickhouse-config.xml - /var/lib/clickhouse - /hdd1_data - /hdd2_data @@ -216,25 +221,23 @@ services: # - ./clickhouse-server.log:/var/log/clickhouse-server/clickhouse-server.log # - ./clickhouse-server.err.log:/var/log/clickhouse-server/clickhouse-server.err.log ports: - - "8123:8123" - - "9000:9000" + - "8123" + - "9000" # for delve debugger - - "40001:40001" - networks: - - clickhouse-backup + - "40002" links: - zookeeper - minio - sshd - ftp - azure -# - gcs + - gcs healthcheck: test: clickhouse client -q "SELECT 1" - interval: 10s + interval: 1s timeout: 2s - retries: 30 - start_period: 5s + retries: 60 + start_period: 1s depends_on: zookeeper: condition: service_healthy @@ -242,6 +245,8 @@ services: condition: service_healthy azure: condition: service_healthy + gcs: + condition: service_healthy # azure_init: # condition: service_completed_successfully @@ -251,5 +256,10 @@ services: clickhouse-backup: condition: service_healthy -networks: - clickhouse-backup: +volumes: + azure: + driver: local + driver_opts: + device: tmpfs + type: tmpfs + o: size=60m \ No newline at end of file diff --git a/test/integration/docker-compose_advanced.yml b/test/integration/docker-compose_advanced.yml index 33b2efdc..bc3b755f 100644 --- a/test/integration/docker-compose_advanced.yml +++ b/test/integration/docker-compose_advanced.yml @@ -1,13 +1,11 @@ services: sshd: image: docker.io/panubo/sshd:latest - container_name: sshd + hostname: sshd environment: SSH_ENABLE_ROOT: "true" SSH_ENABLE_PASSWORD_AUTH: "true" command: sh -c 'echo "PermitRootLogin yes" >> /etc/ssh/sshd_config && echo "LogLevel DEBUG3" >> /etc/ssh/sshd_config && echo "root:JFzMHfVpvTgEd74XXPq6wARA2Qg3AutJ" | chpasswd && /usr/sbin/sshd -D -e -f /etc/ssh/sshd_config' - networks: - - clickhouse-backup # ftp: # image: docker.io/fauria/vsftpd:latest @@ -20,8 +18,6 @@ services: # PASV_ADDR_RESOLVE: "YES" # PASV_MIN_PORT: 21100 # PASV_MAX_PORT: 21110 -# networks: -# - clickhouse-backup ftp: image: docker.io/iradu/proftpd:latest @@ -31,53 +27,58 @@ services: FTP_USER_PASS: "test_backup" FTP_MASQUERADEADDRESS: "yes" FTP_PASSIVE_PORTS: "21100 31100" - networks: - - clickhouse-backup minio: image: docker.io/bitnami/minio:${MINIO_VERSION:-latest} - container_name: minio + hostname: minio environment: - MINIO_ACCESS_KEY: access-key - MINIO_SECRET_KEY: it-is-my-super-secret-key MINIO_DEFAULT_BUCKETS: 'clickhouse' - MINIO_ROOT_USER: access-key - MINIO_ROOT_PASSWORD: it-is-my-super-secret-key + MINIO_ROOT_USER: access_key + MINIO_ROOT_PASSWORD: it_is_my_super_secret_key + MINIO_SCHEME: https + BITNAMI_DEBUG: "true" healthcheck: - test: curl -sL http://localhost:9000/ - interval: 10s - retries: 30 + test: ls -la /bitnami/minio/data/clickhouse/ && curl -skL https://localhost:9000/ + interval: 1s + retries: 60 volumes: - ./minio_nodelete.sh:/bin/minio_nodelete.sh - ports: - - "9001:9001" - networks: - - clickhouse-backup - -# todo need to reproduce download after upload -# gcs: -# image: fsouza/fake-gcs-server:latest -# hostname: gcs -# entrypoint: -# - /bin/sh -# command: -# - -c -# - "mkdir -p /data/clickhouse-backup-test-gcs && fake-gcs-server -data /data -scheme http -port 8080 -public-host gsc:8080" -# networks: -# - clickhouse-backup + # go install github.com/minio/certgen@latest + # ~/go/bin/certgen -host "127.0.0.1,localhost,minio" + - ./minio.crt:/certs/CAs/public.crt + - ./minio.crt:/certs/public.crt + - ./minio.key:/certs/private.key + gcs: + image: fsouza/fake-gcs-server:latest + hostname: gcs + entrypoint: + - /bin/sh + command: + - -c + - "mkdir -p /data/altinity-qa-test && mkdir -p /data/${QA_GCS_OVER_S3_BUCKET} && fake-gcs-server -data /data -scheme http -port 8080 -public-host gcs:8080" + environment: + QA_GCS_OVER_S3_BUCKET: "${QA_GCS_OVER_S3_BUCKET}" + healthcheck: + test: nc 127.0.0.1 8080 -z + interval: 1s + retries: 30 azure: + # todo need to blobKeepAliveTimeout, and merge https://github.com/Azure/Azurite/pull/2443 image: mcr.microsoft.com/azure-storage/azurite:latest - container_name: azure + # image: docker.io/clickhousepro/azurite:latest hostname: devstoreaccount1.blob.azure healthcheck: test: nc 127.0.0.1 10000 -z interval: 1s retries: 30 - command: ["azurite", "--debug", "/data/debug.log" , "-l", "/data", "--blobHost", "0.0.0.0","--queueHost", "0.0.0.0", "--tableHost", "0.0.0.0"] - networks: - - clickhouse-backup + command: [ "azurite", "--debug", "/data/debug.log" , "-l", "/data", "--blobHost", "0.0.0.0" ] + # command: [ "azurite", "--debug", "/data/debug.log" , "-l", "/data", "--blobHost", "0.0.0.0","--blobKeepAliveTimeout", "3600" ] + volumes: + - azure:/data +# environment: +# - AZURITE_DB="mysql://root:root@mysql:3306/azurite_blob" # azure_init: # image: mcr.microsoft.com/azure-cli:latest @@ -93,23 +94,22 @@ services: # environment: # # https://github.com/Azure/Azurite/blob/main/README.md#usage-with-azure-storage-sdks-or-tools # AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure:10000/devstoreaccount1; - # networks: - # - clickhouse-backup mysql: image: docker.io/mysql:${MYSQL_VERSION:-latest} - command: --default-authentication-plugin=mysql_native_password --gtid_mode=on --enforce_gtid_consistency=ON + command: --gtid_mode=on --enforce_gtid_consistency=ON hostname: mysql environment: MYSQL_ROOT_PASSWORD: "root" ports: - - "3306:3306" - networks: - - clickhouse-backup + - "3306" + volumes: + - mysql:/var/lib/mysql healthcheck: test: mysqladmin -p=root ping -h localhost - timeout: 20s - retries: 10 + timeout: 10s + interval: 1s + retries: 100 pgsql: image: docker.io/postgres:${PGSQL_VERSION:-latest} @@ -120,14 +120,15 @@ services: # to allow connection from clickhouse 21.3 POSTGRES_HOST_AUTH_METHOD: "md5" ports: - - "5432:5432" - networks: - - clickhouse-backup + - "5432" command: [ "postgres", "-c", "wal_level=logical" ] healthcheck: test: pg_isready - timeout: 20s - retries: 10 + timeout: 10s + interval: 1s + retries: 60 + volumes: + - pgsql:/var/lib/postgresql zookeeper: image: docker.io/clickhouse/clickhouse-keeper:${CLICKHOUSE_KEEPER_VERSION:-latest-alpine} @@ -135,20 +136,21 @@ services: volumes: - ./clickhouse-keeper.xml:/etc/clickhouse-keeper/conf.d/clickhouse-keeper.xml - /var/lib/clickhouse - networks: - - clickhouse-backup + - /var/lib/clickhouse-keeper + environment: + - CLICKHOUSE_UID=0 + - CLICKHOUSE_GID=0 healthcheck: test: bash -c 'if [[ "$$(echo 'ruok' | nc 127.0.0.1 2181)" == "imok" ]]; then exit 0; else exit 1; fi' - interval: 3s + interval: 1s timeout: 2s - retries: 5 - start_period: 2s + retries: 10 + start_period: 1s clickhouse-backup: image: docker.io/${CLICKHOUSE_IMAGE:-yandex/clickhouse-server}:${CLICKHOUSE_VERSION:-19.17} hostname: clickhouse-backup - container_name: clickhouse-backup user: root entrypoint: - /bin/bash @@ -156,7 +158,7 @@ services: - sleep infinity healthcheck: test: bash -c "exit 0" - interval: 30s + interval: 1s timeout: 1s retries: 5 start_period: 1s @@ -168,27 +170,27 @@ services: GCS_DEBUG: "${GCS_DEBUG:-false}" FTP_DEBUG: "${FTP_DEBUG:-false}" SFTP_DEBUG: "${SFTP_DEBUG:-false}" + AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}" CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}" GOCOVERDIR: "/tmp/_coverage_/" -# fake-gcs-server -# STORAGE_EMULATOR_HOST: "http://gsc:8080" -# GOOGLE_API_USE_CLIENT_CERTIFICATE: "false" # FIPS QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY} QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY} QA_AWS_BUCKET: ${QA_AWS_BUCKET} QA_AWS_REGION: ${QA_AWS_REGION} +# GCS over S3 embedded backups + QA_GCS_OVER_S3_ACCESS_KEY: "${QA_GCS_OVER_S3_ACCESS_KEY}" + QA_GCS_OVER_S3_SECRET_KEY: "${QA_GCS_OVER_S3_SECRET_KEY}" + QA_GCS_OVER_S3_BUCKET: "${QA_GCS_OVER_S3_BUCKET}" # https://github.com/Altinity/clickhouse-backup/issues/691: - AWS_ACCESS_KEY_ID: access-key - AWS_SECRET_ACCESS_KEY: it-is-my-super-secret-key + AWS_ACCESS_KEY_ID: access_key + AWS_SECRET_ACCESS_KEY: it_is_my_super_secret_key volumes_from: - clickhouse ports: - - "7171:7171" + - "7171" # for delve debugger - - "40001:40001" - networks: - - clickhouse-backup + - "40001" depends_on: clickhouse: condition: service_healthy @@ -196,35 +198,40 @@ services: clickhouse: image: docker.io/${CLICKHOUSE_IMAGE:-yandex/clickhouse-server}:${CLICKHOUSE_VERSION:-19.17} hostname: clickhouse - container_name: clickhouse restart: always user: root environment: CLICKHOUSE_VERSION: ${CLICKHOUSE_VERSION:-19.17} + CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS: "true" TZ: UTC LOG_LEVEL: "${LOG_LEVEL:-info}" S3_DEBUG: "${S3_DEBUG:-false}" GCS_DEBUG: "${GCS_DEBUG:-false}" FTP_DEBUG: "${FTP_DEBUG:-false}" SFTP_DEBUG: "${SFTP_DEBUG:-false}" + AZBLOB_DEBUG: "${AZBLOB_DEBUG:-false}" CLICKHOUSE_DEBUG: "${CLICKHOUSE_DEBUG:-false}" GOCOVERDIR: "/tmp/_coverage_/" -# fake-gcs-server -# STORAGE_EMULATOR_HOST: "http://gsc:8080" -# GOOGLE_API_USE_CLIENT_CERTIFICATE: "false" # FIPS QA_AWS_ACCESS_KEY: ${QA_AWS_ACCESS_KEY} QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY} QA_AWS_BUCKET: ${QA_AWS_BUCKET} QA_AWS_REGION: ${QA_AWS_REGION} # https://github.com/Altinity/clickhouse-backup/issues/691: - AWS_ACCESS_KEY_ID: access-key - AWS_SECRET_ACCESS_KEY: it-is-my-super-secret-key + AWS_ACCESS_KEY_ID: access_key + AWS_SECRET_ACCESS_KEY: it_is_my_super_secret_key # GCS over S3 object disk QA_GCS_OVER_S3_ACCESS_KEY: "${QA_GCS_OVER_S3_ACCESS_KEY}" QA_GCS_OVER_S3_SECRET_KEY: "${QA_GCS_OVER_S3_SECRET_KEY}" QA_GCS_OVER_S3_BUCKET: "${QA_GCS_OVER_S3_BUCKET}" AWS_EC2_METADATA_DISABLED: "true" + + # to avoid backward incompatibility ;( + # https://t.me/clickhouse_ru/359960 + # https://t.me/clickhouse_ru/359968 + # https://t.me/clickhouse_ru/362378 + entrypoint: + - "/custom_entrypoint.sh" volumes: # clickhouse-backup related files requires for some tests - ${CLICKHOUSE_BACKUP_BIN:-../../clickhouse-backup/clickhouse-backup-race}:/usr/bin/clickhouse-backup @@ -232,14 +239,20 @@ services: - ./credentials.json:/etc/clickhouse-backup/credentials.json - ./config-azblob.yml:/etc/clickhouse-backup/config-azblob.yml - ./config-azblob-embedded.yml:/etc/clickhouse-backup/config-azblob-embedded.yml + - ./config-azblob-embedded-url.yml:/etc/clickhouse-backup/config-azblob-embedded-url.yml - ./config-custom-kopia.yml:/etc/clickhouse-backup/config-custom-kopia.yml - ./config-custom-restic.yml:/etc/clickhouse-backup/config-custom-restic.yml - ./config-custom-rsync.yml:/etc/clickhouse-backup/config-custom-rsync.yml - ./config-database-mapping.yml:/etc/clickhouse-backup/config-database-mapping.yml - ./config-ftp.yaml:/etc/clickhouse-backup/config-ftp.yaml + - ./config-ftp-old.yaml:/etc/clickhouse-backup/config-ftp-old.yaml - ./config-gcs.yml:/etc/clickhouse-backup/config-gcs.yml + - ./config-gcs-embedded-url.yml:/etc/clickhouse-backup/config-gcs-embedded-url.yml.template + - ./config-gcs-custom-endpoint.yml:/etc/clickhouse-backup/config-gcs-custom-endpoint.yml - ./config-s3.yml:/etc/clickhouse-backup/config-s3.yml - ./config-s3-embedded.yml:/etc/clickhouse-backup/config-s3-embedded.yml + - ./config-s3-embedded-url.yml:/etc/clickhouse-backup/config-s3-embedded-url.yml + - ./config-s3-embedded-local.yml:/etc/clickhouse-backup/config-s3-embedded-local.yml - ./config-s3-fips.yml:/etc/clickhouse-backup/config-s3-fips.yml.template - ./config-s3-nodelete.yml:/etc/clickhouse-backup/config-s3-nodelete.yml - ./config-s3-plain-embedded.yml:/etc/clickhouse-backup/config-s3-plain-embedded.yml @@ -249,6 +262,7 @@ services: # for local debug - ./install_delve.sh:/tmp/install_delve.sh # clickhouse configuration + - ./custom_entrypoint.sh:/custom_entrypoint.sh - ./dynamic_settings.sh:/docker-entrypoint-initdb.d/dynamic_settings.sh - ./enable-access_management.xml:/etc/clickhouse-server/users.d/enable-access_management.xml - ./backup-user.xml:/etc/clickhouse-server/users.d/backup-user.xml @@ -256,7 +270,7 @@ services: - ./server.key:/etc/clickhouse-server/server.key - ./dhparam.pem:/etc/clickhouse-server/dhparam.pem - ./ssl.xml:/etc/clickhouse-server/config.d/ssl.xml - - ./cluster.xml:/etc/clickhouse-server/config.d/cluster.xml + - ./clickhouse-config.xml:/etc/clickhouse-server/config.d/clickhouse-config.xml - /var/lib/clickhouse - /hdd1_data - /hdd2_data @@ -265,12 +279,10 @@ services: # - ./clickhouse-server.log:/var/log/clickhouse-server/clickhouse-server.log # - ./clickhouse-server.err.log:/var/log/clickhouse-server/clickhouse-server.err.log ports: - - "8123:8123" - - "9000:9000" + - "8123" + - "9000" # for delve debugger -# - "40001:40001" - networks: - - clickhouse-backup + - "40002" links: - zookeeper - minio @@ -279,13 +291,13 @@ services: - pgsql - ftp - azure -# - gcs + - gcs healthcheck: test: clickhouse client -q "SELECT 1" - interval: 10s + interval: 1s timeout: 2s - retries: 30 - start_period: 5s + retries: 60 + start_period: 1s depends_on: mysql: condition: service_healthy @@ -297,6 +309,8 @@ services: condition: service_healthy azure: condition: service_healthy + gcs: + condition: service_healthy # azure_init: # condition: service_completed_successfully @@ -306,5 +320,22 @@ services: clickhouse-backup: condition: service_healthy -networks: - clickhouse-backup: +volumes: + mysql: + driver: local + driver_opts: + device: tmpfs + type: tmpfs + o: size=250m + pgsql: + driver: local + driver_opts: + device: tmpfs + type: tmpfs + o: size=60m + azure: + driver: local + driver_opts: + device: tmpfs + type: tmpfs + o: size=60m \ No newline at end of file diff --git a/test/integration/dynamic_settings.sh b/test/integration/dynamic_settings.sh old mode 100644 new mode 100755 index 35828c5c..8a970c0d --- a/test/integration/dynamic_settings.sh +++ b/test/integration/dynamic_settings.sh @@ -44,7 +44,7 @@ cat < /etc/clickhouse-server/config.d/storage_configuration.xml EOT -if [[ "${CLICKHOUSE_VERSION}" =~ ^21\.1[0-9] || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[0-9]+ ]]; then +if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.1[0-9] || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[0-9]+ ]]; then if [[ ! -d /hdd3_data ]]; then mkdir -pv /hdd3_data @@ -82,17 +82,19 @@ EOT fi if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.[8-9]|^21\.[0-9]{2} || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[0-9]+ ]]; then - +if [[ -f /var/lib/clickhouse/storage_configuration_s3.xml ]]; then + cp -fv /var/lib/clickhouse/storage_configuration_s3.xml /etc/clickhouse-server/config.d/storage_configuration_s3.xml +else cat < /etc/clickhouse-server/config.d/storage_configuration_s3.xml s3 - http://minio:9000/clickhouse/disk_s3/ + https://minio:9000/clickhouse/disk_s3/{cluster}/{shard}/ 1 @@ -112,31 +114,35 @@ cat < /etc/clickhouse-server/config.d/storage_configuration_s3.xml EOT +fi fi if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^22\.[6-9]+ || "${CLICKHOUSE_VERSION}" =~ ^22\.1[0-9]+ || "${CLICKHOUSE_VERSION}" =~ ^2[3-9]\.[1-9]+ ]]; then if [[ "" != "${QA_GCS_OVER_S3_BUCKET}" ]]; then +if [[ -f /var/lib/clickhouse/storage_configuration_gcs.xml ]]; then + cp -fv /var/lib/clickhouse/storage_configuration_gcs.xml /etc/clickhouse-server/config.d/storage_configuration_gcs.xml +else cat < /etc/clickhouse-server/config.d/storage_configuration_gcs.xml - + s3 - https://storage.googleapis.com/${QA_GCS_OVER_S3_BUCKET}/clickhouse_backup_disk_gcs_over_s3/${HOSTNAME}/ + https://storage.googleapis.com/${QA_GCS_OVER_S3_BUCKET}/clickhouse_backup_disk_gcs_over_s3/${HOSTNAME}/{cluster}/{shard}/ ${QA_GCS_OVER_S3_ACCESS_KEY} ${QA_GCS_OVER_S3_SECRET_KEY} false false - + - disk_gcs_over_s3 + disk_gcs @@ -145,22 +151,27 @@ cat < /etc/clickhouse-server/config.d/storage_configuration_gcs.xml EOT +fi fi fi if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.12 || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[0-9]+ ]]; then +if [[ -f /var/lib/clickhouse/storage_configuration_encrypted_s3.xml ]]; then + cp -fv /var/lib/clickhouse/storage_configuration_encrypted_s3.xml /etc/clickhouse-server/config.d/storage_configuration_encrypted_s3.xml +else + cat < /etc/clickhouse-server/config.d/storage_configuration_encrypted_s3.xml s3 - http://minio:9000/clickhouse/disk_s3/ + https://minio:9000/clickhouse/disk_s3/ 1 @@ -191,6 +202,33 @@ cat < /etc/clickhouse-server/config.d/storage_configuration_encrypted_s3.x EOT +fi +fi + +# embedded local backup configuration +if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^22\.[6-9] || "${CLICKHOUSE_VERSION}" =~ ^22\.1[0-9]+ || "${CLICKHOUSE_VERSION}" =~ ^2[3-9]\.[0-9]+ ]]; then + +mkdir -p /var/lib/clickhouse/disks/backups_local/ /var/lib/clickhouse/backups_embedded/ +chown -R clickhouse /var/lib/clickhouse/disks/ /var/lib/clickhouse/backups_embedded/ + +cat < /etc/clickhouse-server/config.d/backup_storage_configuration_local.xml + + + + + + local + /var/lib/clickhouse/disks/backups_local/ + + + + + backups_local + /var/lib/clickhouse/backups_embedded/ + + +EOT + fi # embedded s3 backup configuration @@ -206,10 +244,10 @@ cat < /etc/clickhouse-server/config.d/backup_storage_configuration_s3.xml s3 - http://minio:9000/clickhouse/backups_s3/ + https://minio:9000/clickhouse/backups_s3/{cluster}/{shard}/ 1 false @@ -219,15 +257,38 @@ cat < /etc/clickhouse-server/config.d/backup_storage_configuration_s3.xml + backups_local backups_s3 /var/lib/clickhouse/backups_embedded/ - - 1 - EOT +# zero replication is buggy, can't freeze table: code: 344, message: FREEZE PARTITION queries are disabled. +# https://github.com/ClickHouse/ClickHouse/issues/62167#issuecomment-2031774983 +#cat < /etc/clickhouse-server/config.d/zero_copy_replication.xml +# +# +# 1 +# 0 +# +# +#EOT + +cat < /etc/clickhouse-server/config.d/zookeeper_log.xml + + + system +
zookeeper_log
+ 7500 + 1048576 + 8192 + 524288 + 1000 + true + + +EOT fi # s3_plain and azure backup configuration @@ -243,10 +304,10 @@ cat < /etc/clickhouse-server/config.d/backup_storage_configuration_s3_plai s3_plain - http://minio:9000/clickhouse/backups_s3_plain/ + https://minio:9000/clickhouse/backups_s3_plain/{cluster}/{shard}/ 1 false @@ -254,6 +315,7 @@ cat < /etc/clickhouse-server/config.d/backup_storage_configuration_s3_plai + backups_local backups_s3 backups_s3_plain @@ -263,12 +325,16 @@ EOT mkdir -p /var/lib/clickhouse/disks/backups_azure/ chown -R clickhouse /var/lib/clickhouse/disks/ -cat < /etc/clickhouse-server/config.d/backup_storage_configuration_azure.xml +if [[ -f /var/lib/clickhouse/storage_configuration_azblob.xml ]]; then + cp -fv /var/lib/clickhouse/storage_configuration_azblob.xml /etc/clickhouse-server/config.d/backup_storage_configuration_azblob.xml +else + +cat < /etc/clickhouse-server/config.d/storage_configuration_azblob.xml - + azure_blob_storage http://azure:10000/devstoreaccount1 azure-disk @@ -276,7 +342,8 @@ cat < /etc/clickhouse-server/config.d/backup_storage_configuration_azure.x devstoreaccount1 Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== false - + true + azure_blob_storage http://azure:10000/devstoreaccount1 @@ -285,25 +352,28 @@ cat < /etc/clickhouse-server/config.d/backup_storage_configuration_azure.x devstoreaccount1 Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== false + true - azure + disk_azblob + backups_local backups_s3 backups_s3_plain backups_azure EOT +fi fi @@ -359,6 +429,9 @@ fi # zookeeper RBAC available from 21.9 if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^21\.9 || "${CLICKHOUSE_VERSION}" =~ ^21\.1[0-9] || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[1-9] ]]; then +mkdir -p /var/lib/clickhouse/access +chown clickhouse:clickhouse /var/lib/clickhouse/access + cat < /etc/clickhouse-server/config.d/replicated_user_directories.xml @@ -372,4 +445,49 @@ cat < /etc/clickhouse-server/config.d/replicated_user_directories.xml EOT +fi + +# @todo LIVE VIEW deprecated, available 21.3+ +if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[1-9] || "${CLICKHOUSE_VERSION}" =~ ^21\.[3-9] || "${CLICKHOUSE_VERSION}" =~ ^21\.1[0-9] ]]; then + +cat < /etc/clickhouse-server/users.d/allow_experimental_live_view.xml + + + 1 + + +EOT + +fi + +# WINDOW VIEW available 21.12+ +if [[ "${CLICKHOUSE_VERSION}" == "head" || "${CLICKHOUSE_VERSION}" =~ ^2[2-9]\.[1-9] || "${CLICKHOUSE_VERSION}" =~ ^21\.1[1-2] ]]; then + +cat < /etc/clickhouse-server/users.d/allow_experimental_window_view.xml + + + 1 + + +EOT + +fi + +# blob_storage_log available in 23.11 +if [[ "$CLICKHOUSE_VERSION" == "head" || "${CLICKHOUSE_VERSION}" =~ ^23\.1[1-9] || "${CLICKHOUSE_VERSION}" =~ ^2[4-9\.[1-9] ]]; then + +cat < /etc/clickhouse-server/config.d/blob_storage_log.xml + + + system + blob_storage_log
+ ENGINE = MergeTree PARTITION BY (event_date) + ORDER BY (event_time) + TTL event_date + INTERVAL 1 DAY DELETE + + 7500 + + +EOT + fi \ No newline at end of file diff --git a/test/integration/install_delve.sh b/test/integration/install_delve.sh index 4217e679..227c314c 100755 --- a/test/integration/install_delve.sh +++ b/test/integration/install_delve.sh @@ -4,21 +4,24 @@ apt-get update && apt-get install -y software-properties-common apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 52B59B1571A79DBC054901C0F6BC817356A3D45E add-apt-repository -y ppa:longsleep/golang-backports apt-get update -apt-get install -y golang-1.21 +apt-get install -y golang-1.22 mkdir -p ~/go/ export GOPATH=~/go/ grep -q -F 'export GOPATH=$GOPATH' ~/.bashrc || echo "export GOPATH=$GOPATH" >> ~/.bashrc grep -q -F 'export GOPATH=$GOPATH' /root/.bashrc || echo "export GOPATH=$GOPATH" >> /root/.bashrc -export GOROOT=/usr/lib/go-1.21/ +export GOROOT=/usr/lib/go-1.22/ grep -q -F 'export GOROOT=$GOROOT' ~/.bashrc || echo "export GOROOT=$GOROOT" >> ~/.bashrc grep -q -F 'export GOROOT=$GOROOT' /root/.bashrc || echo "export GOROOT=$GOROOT" >> /root/.bashrc -ln -nsfv /usr/lib/go-1.21/bin/go /usr/bin/go +ln -nsfv /usr/lib/go-1.22/bin/go /usr/bin/go CGO_ENABLED=0 GO111MODULE=on go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest # GO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -tags osusergo,netgo -gcflags "all=-N -l" -ldflags "-extldflags '-static' -X 'main.version=debug'" -o build/linux/amd64/clickhouse-backup ./cmd/clickhouse-backup +# /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- -c /etc/clickhouse-backup/config-azblob.yml download --partitions=test_partitions_TestIntegrationAzure.t?:(0,'2022-01-02'),(0,'2022-01-03') full_backup_3691696362844433277 +# /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- -c /etc/clickhouse-backup/config-azblob.yml restore --schema TestIntegrationAzure_full_6516689450475708573 # /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- -c /etc/clickhouse-server/config.d/ch-backup.yaml upload debug_upload --table +# USE_RESUMABLE_STATE=0 CLICKHOUSE_SKIP_TABLES=*.test_memory /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- -c /etc/clickhouse-backup/config-s3.yml download test_skip_full_backup # /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- download test_rbac_backup # /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- download keep_remote_backup_4 # /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- server @@ -26,9 +29,13 @@ CGO_ENABLED=0 GO111MODULE=on go install -ldflags "-s -w -extldflags '-static'" g # CLICKHOUSE_TIMEOUT=3m CLICKHOUSE_DEBUG=true LOG_LEVEL=debug /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /usr/bin/clickhouse-backup -- watch --watch-interval=1m --full-interval=2m # LOG_LEVEL=debug /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /usr/bin/clickhouse-backup -- restore --data --restore-database-mapping database1:database2 --tables database1.* test_restore_database_mapping # S3_DEBUG=true /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /usr/bin/clickhouse-backup -- upload test_rbac_backup +# S3_DEBUG=true /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /usr/bin/clickhouse-backup -- -c /etc/clickhouse-backup/config-s3.yml delete remote full_backup_339504125792808941 +# /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /usr/bin/clickhouse-backup -- -c /etc/clickhouse-backup/config-custom-kopia.yml list remote # run integration_test.go under debug, run from host OS not inside docker # go test -timeout 30m -failfast -tags=integration -run "TestIntegrationEmbedded" -v ./test/integration/integration_test.go -c -o ./test/integration/integration_test # sudo -H bash -c 'export CLICKHOUSE_IMAGE=clickhouse/clickhouse-server; export COMPOSE_FILE=docker-compose_advanced.yml; export CLICKHOUSE_VERSION=head; cd ./test/integration/; /root/go/bin/dlv --listen=127.0.0.1:40002 --headless=true --api-version=2 --accept-multiclient exec ./integration_test -- -test.timeout 30m -test.failfast -test.run "TestIntegrationEmbedded"' +# /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- -c /etc/clickhouse-backup/config-s3.yml download --partitions="test_partitions_TestIntegrationS3.t?:(0,'2022-01-02'),(0,'2022-01-03')" full_backup_5643339940028285692 +# EMBEDDED_S3_COMPRESSION_FORMAT=zstd CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3-embedded.yml /root/go/bin/dlv --listen=:40001 --headless=true --api-version=2 --accept-multiclient exec /bin/clickhouse-backup -- upload TestIntegrationEmbedded_full_5990789107828261693 diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 987d98c7..5c99534a 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -3,39 +3,85 @@ package main import ( + "bufio" "context" + "encoding/json" "fmt" - "github.com/Altinity/clickhouse-backup/pkg/config" - "github.com/Altinity/clickhouse-backup/pkg/log_helper" - "github.com/Altinity/clickhouse-backup/pkg/partition" - "github.com/Altinity/clickhouse-backup/pkg/status" - "github.com/Altinity/clickhouse-backup/pkg/utils" - "github.com/google/uuid" - "github.com/rs/zerolog" - "github.com/rs/zerolog/pkgerrors" - stdlog "log" + pool "github.com/jolestar/go-commons-pool/v2" "math/rand" "os" "os/exec" + "path" "reflect" "regexp" + "slices" "strconv" "strings" "sync" + "sync/atomic" "testing" "time" - "github.com/Altinity/clickhouse-backup/pkg/clickhouse" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" + "github.com/rs/zerolog/pkgerrors" "golang.org/x/mod/semver" + stdlog "log" _ "github.com/ClickHouse/clickhouse-go/v2" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/Altinity/clickhouse-backup/v2/pkg/clickhouse" + "github.com/Altinity/clickhouse-backup/v2/pkg/config" + "github.com/Altinity/clickhouse-backup/v2/pkg/log_helper" + "github.com/Altinity/clickhouse-backup/v2/pkg/partition" + "github.com/Altinity/clickhouse-backup/v2/pkg/status" + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" ) -const dbNameAtomic = "_test.ДБ_atomic_" -const dbNameOrdinary = "_test.ДБ_ordinary_" +var projectId atomic.Uint32 +var dockerPool *pool.ObjectPool + +// setup log level +func init() { + zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs + zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack + consoleWriter := zerolog.ConsoleWriter{Out: os.Stderr, NoColor: true, TimeFormat: "2006-01-02 15:04:05.000"} + log.Logger = zerolog.New(zerolog.SyncWriter(consoleWriter)).With().Timestamp().Logger() + stdlog.SetOutput(log.Logger) + logLevel := "info" + if os.Getenv("LOG_LEVEL") != "" && os.Getenv("LOG_LEVEL") != "info" { + logLevel = os.Getenv("LOG_LEVEL") + } + if os.Getenv("TEST_LOG_LEVEL") != "" && os.Getenv("TEST_LOG_LEVEL") != "info" { + logLevel = os.Getenv("TEST_LOG_LEVEL") + } + log_helper.SetLogLevelFromString(logLevel) + + runParallel, isExists := os.LookupEnv("RUN_PARALLEL") + if !isExists { + runParallel = "1" + } + runParallelInt, err := strconv.Atoi(runParallel) + if err != nil { + log.Fatal().Msgf("invalid RUN_PARALLEL environment variable value %s", runParallel) + } + ctx := context.Background() + factory := pool.NewPooledObjectFactorySimple( func(context.Context) (interface{}, error) { + id := projectId.Add(1) + env := TestEnvironment{ + ProjectName: fmt.Sprintf("project%d", id%uint32(runParallelInt)), + } + return &env, nil + }) + dockerPool = pool.NewObjectPoolWithDefaultConfig(ctx, factory) + dockerPool.Config.MaxTotal = runParallelInt +} + +const dbNameAtomic = "_test#$.ДБ_atomic_" +const dbNameOrdinary = "_test#$.ДБ_ordinary_" const dbNameMySQL = "mysql_db" const dbNamePostgreSQL = "pgsql_db" const Issue331Atomic = "_issue331._atomic_" @@ -57,9 +103,15 @@ type TestDataStruct struct { CheckDatabaseOnly bool } +type TestEnvironment struct { + ch *clickhouse.ClickHouse + ProjectName string +} + var defaultTestData = []TestDataStruct{ { Database: dbNameOrdinary, DatabaseEngine: "Ordinary", + // 24.8 shall resolve https://github.com/ClickHouse/ClickHouse/issues/67669 Name: ".inner.table1", Schema: "(Date Date, TimeStamp DateTime, Logger String) ENGINE = MergeTree(Date, (TimeStamp, Logger), 8192)", Rows: []map[string]interface{}{ @@ -88,7 +140,7 @@ var defaultTestData = []TestDataStruct{ OrderBy: "id", }, { Database: dbNameOrdinary, DatabaseEngine: "Ordinary", - Name: "-table-3-", + Name: "-table-$-", Schema: "(TimeStamp DateTime, Item String, Date Date MATERIALIZED toDate(TimeStamp)) ENGINE = MergeTree() PARTITION BY Date ORDER BY TimeStamp SETTINGS index_granularity = 8192", Rows: []map[string]interface{}{ {"TimeStamp": toTS("2018-10-23 07:37:14"), "Item": "One"}, @@ -145,7 +197,7 @@ var defaultTestData = []TestDataStruct{ OrderBy: "order_id", }, { Database: dbNameOrdinary, DatabaseEngine: "Ordinary", - Name: "jbod_table", + Name: "jbod#$_table", Schema: "(id UInt64) Engine=MergeTree ORDER BY id SETTINGS storage_policy = 'jbod'", Rows: func() []map[string]interface{} { var result []map[string]interface{} @@ -158,7 +210,7 @@ var defaultTestData = []TestDataStruct{ OrderBy: "id", }, { Database: dbNameAtomic, DatabaseEngine: "Atomic", - Name: "jbod_table", + Name: "jbod#$_table", Schema: "(t DateTime, id UInt64) Engine=MergeTree PARTITION BY (toYYYYMM(t), id % 4) ORDER BY id SETTINGS storage_policy = 'jbod'", Rows: func() []map[string]interface{} { var result []map[string]interface{} @@ -307,6 +359,7 @@ var defaultTestData = []TestDataStruct{ var defaultIncrementData = []TestDataStruct{ { Database: dbNameOrdinary, DatabaseEngine: "Ordinary", + // 24.8 shall resolve https://github.com/ClickHouse/ClickHouse/issues/67669 Name: ".inner.table1", Schema: "(Date Date, TimeStamp DateTime, Logger String) ENGINE = MergeTree(Date, (TimeStamp, Logger), 8192)", Rows: []map[string]interface{}{ @@ -328,7 +381,7 @@ var defaultIncrementData = []TestDataStruct{ OrderBy: "id", }, { Database: dbNameOrdinary, DatabaseEngine: "Ordinary", - Name: "-table-3-", + Name: "-table-$-", Schema: "(TimeStamp DateTime, Item String, Date Date MATERIALIZED toDate(TimeStamp)) ENGINE = MergeTree() PARTITION BY Date ORDER BY TimeStamp SETTINGS index_granularity = 8192", Rows: []map[string]interface{}{ {"TimeStamp": toTS("2019-01-26 07:37:18"), "Item": "Seven"}, @@ -381,7 +434,7 @@ var defaultIncrementData = []TestDataStruct{ OrderBy: "order_id", }, { Database: dbNameAtomic, DatabaseEngine: "Atomic", - Name: "jbod_table", + Name: "jbod#$_table", Schema: "(t DateTime, id UInt64) Engine=MergeTree PARTITION BY (toYYYYMM(t), id % 4) ORDER BY id SETTINGS storage_policy = 'jbod'", Rows: func() []map[string]interface{} { var result []map[string]interface{} @@ -395,299 +448,544 @@ var defaultIncrementData = []TestDataStruct{ }, } -func init() { - zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs - zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack - consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout, NoColor: true, TimeFormat: "2006-01-02 15:04:05.000"} - log.Logger = zerolog.New(zerolog.SyncWriter(consoleWriter)).With().Timestamp().Logger() - stdlog.SetOutput(log.Logger) - logLevel := "info" - if os.Getenv("LOG_LEVEL") != "" { - logLevel = os.Getenv("LOG_LEVEL") +func NewTestEnvironment(t *testing.T) (*TestEnvironment, *require.Assertions) { + isParallel := os.Getenv("RUN_PARALLEL") != "1" && slices.Index([]string{"TestLongListRemote"/*,"TestIntegrationAzure"*/}, t.Name()) == -1 + if os.Getenv("COMPOSE_FILE") == "" || os.Getenv("CUR_DIR") == "" { + t.Fatal("please setup COMPOSE_FILE and CUR_DIR environment variables") } - log_helper.SetLogLevelFromString(logLevel) - r := require.New(&testing.T{}) - installDebIfNotExists(r, "clickhouse-backup", "ca-certificates", "curl") - r.NoError(dockerExec("clickhouse-backup", "update-ca-certificates")) - r.NoError(dockerExec("clickhouse-backup", "bash", "-xce", "curl -sL \"https://github.com/mikefarah/yq/releases/latest/download/yq_linux_$(dpkg --print-architecture)\" -o /usr/bin/yq && chmod +x /usr/bin/yq")) - installDebIfNotExists(r, "clickhouse-backup", "jq", "bzip2", "pgp", "git") - // rsync - installDebIfNotExists(r, "clickhouse-backup", "openssh-client", "rsync") - // kopia - r.NoError(dockerExec("clickhouse-backup", "bash", "-ce", "curl -sfL https://kopia.io/signing-key | gpg --dearmor -o /usr/share/keyrings/kopia-keyring.gpg")) - r.NoError(dockerExec("clickhouse-backup", "bash", "-ce", "echo 'deb [signed-by=/usr/share/keyrings/kopia-keyring.gpg] https://packages.kopia.io/apt/ stable main' > /etc/apt/sources.list.d/kopia.list")) - installDebIfNotExists(r, "clickhouse-backup", "kopia") - // restic - r.NoError(dockerExec("clickhouse-backup", "bash", "-xec", "RELEASE_TAG=$(curl -H 'Accept: application/json' -sL https://github.com/restic/restic/releases/latest | jq -c -r -M '.tag_name'); RELEASE=$(echo ${RELEASE_TAG} | sed -e 's/v//'); curl -sfL \"https://github.com/restic/restic/releases/download/${RELEASE_TAG}/restic_${RELEASE}_linux_amd64.bz2\" | bzip2 -d > /bin/restic; chmod +x /bin/restic")) -} - -// TestS3NoDeletePermission - no parallel -func TestS3NoDeletePermission(t *testing.T) { - if isTestShouldSkip("RUN_ADVANCED_TESTS") { - t.Skip("Skipping Advanced integration tests...") - return + t.Helper() + if isParallel { + t.Parallel() } - r := require.New(t) - r.NoError(dockerExec("minio", "/bin/minio_nodelete.sh")) - r.NoError(dockerCP("config-s3-nodelete.yml", "clickhouse-backup:/etc/clickhouse-backup/config.yml")) - - ch := &TestClickHouse{} - ch.connectWithWait(r, 500*time.Millisecond, 2*time.Second) - defer ch.chbackend.Close() - generateTestData(t, r, ch, "S3", defaultTestData) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "create_remote", "no_delete_backup")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "delete", "local", "no_delete_backup")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "restore_remote", "no_delete_backup")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "delete", "local", "no_delete_backup")) - r.Error(dockerExec("clickhouse-backup", "clickhouse-backup", "delete", "remote", "no_delete_backup")) - databaseList := []string{dbNameOrdinary, dbNameAtomic, dbNameMySQL, dbNamePostgreSQL, Issue331Atomic, Issue331Ordinary} - dropDatabasesFromTestDataDataSet(t, r, ch, databaseList) - r.NoError(dockerExec("minio", "bash", "-ce", "rm -rf /data/clickhouse/*")) -} -// TestDoRestoreRBAC need clickhouse-server restart, no parallel -func TestDoRestoreRBAC(t *testing.T) { - if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.4") == -1 { - t.Skipf("Test skipped, RBAC not available for %s version", os.Getenv("CLICKHOUSE_VERSION")) - } - ch := &TestClickHouse{} r := require.New(t) - - ch.connectWithWait(r, 1*time.Second, 1*time.Second) - - ch.queryWithNoError(r, "DROP TABLE IF EXISTS default.test_rbac") - ch.queryWithNoError(r, "CREATE TABLE default.test_rbac (v UInt64) ENGINE=MergeTree() ORDER BY tuple()") - - ch.queryWithNoError(r, "DROP SETTINGS PROFILE IF EXISTS test_rbac") - ch.queryWithNoError(r, "DROP QUOTA IF EXISTS test_rbac") - ch.queryWithNoError(r, "DROP ROW POLICY IF EXISTS test_rbac ON default.test_rbac") - ch.queryWithNoError(r, "DROP ROLE IF EXISTS test_rbac") - ch.queryWithNoError(r, "DROP USER IF EXISTS test_rbac") - - log.Info().Msg("create RBAC related objects") - ch.queryWithNoError(r, "CREATE SETTINGS PROFILE test_rbac SETTINGS max_execution_time=60") - ch.queryWithNoError(r, "CREATE ROLE test_rbac SETTINGS PROFILE 'test_rbac'") - ch.queryWithNoError(r, "CREATE USER test_rbac IDENTIFIED BY 'test_rbac' DEFAULT ROLE test_rbac") - ch.queryWithNoError(r, "CREATE QUOTA test_rbac KEYED BY user_name FOR INTERVAL 1 hour NO LIMITS TO test_rbac") - ch.queryWithNoError(r, "CREATE ROW POLICY test_rbac ON default.test_rbac USING 1=1 AS RESTRICTIVE TO test_rbac") - - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "--rbac", "--rbac-only", "test_rbac_backup")) - r.NoError(dockerExec("clickhouse-backup", "bash", "-xec", "ALLOW_EMPTY_BACKUPS=1 CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml clickhouse-backup upload test_rbac_backup")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_rbac_backup")) - r.NoError(dockerExec("clickhouse", "ls", "-lah", "/var/lib/clickhouse/access")) - - log.Info().Msg("drop all RBAC related objects after backup") - ch.queryWithNoError(r, "DROP SETTINGS PROFILE test_rbac") - ch.queryWithNoError(r, "DROP QUOTA test_rbac") - ch.queryWithNoError(r, "DROP ROW POLICY test_rbac ON default.test_rbac") - ch.queryWithNoError(r, "DROP ROLE test_rbac") - ch.queryWithNoError(r, "DROP USER test_rbac") - - log.Info().Msg("download+restore RBAC") - r.NoError(dockerExec("clickhouse", "ls", "-lah", "/var/lib/clickhouse/access")) - r.NoError(dockerExec("clickhouse-backup", "bash", "-xec", "ALLOW_EMPTY_BACKUPS=1 CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml clickhouse-backup download test_rbac_backup")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "--rm", "--rbac", "--rbac-only", "test_rbac_backup")) - r.NoError(dockerExec("clickhouse", "ls", "-lah", "/var/lib/clickhouse/access")) - - ch.chbackend.Close() - ch.connectWithWait(r, 2*time.Second, 8*time.Second) - - r.NoError(dockerExec("clickhouse", "ls", "-lah", "/var/lib/clickhouse/access")) - - rbacTypes := map[string]string{ - "PROFILES": "test_rbac", - "QUOTAS": "test_rbac", - "POLICIES": "test_rbac ON default.test_rbac", - "ROLES": "test_rbac", - "USERS": "test_rbac", - } - for rbacType, expectedValue := range rbacTypes { - var rbacRows []struct { - Name string `ch:"name"` - } - err := ch.chbackend.Select(&rbacRows, fmt.Sprintf("SHOW %s", rbacType)) - r.NoError(err) - found := false - for _, row := range rbacRows { - if expectedValue == row.Name { - found = true - break - } - } - if !found { - //r.NoError(dockerExec("clickhouse", "cat", "/var/log/clickhouse-server/clickhouse-server.log")) - r.Failf("wrong RBAC", "SHOW %s, %#v doesn't contain %#v", rbacType, rbacRows, expectedValue) - } + envObj, err := dockerPool.BorrowObject(context.Background()) + if err != nil { + t.Fatalf("dockerPool.BorrowObject retrun error: %v", err) } - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_rbac_backup")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "remote", "test_rbac_backup")) + env := envObj.(*TestEnvironment) - ch.queryWithNoError(r, "DROP SETTINGS PROFILE test_rbac") - ch.queryWithNoError(r, "DROP QUOTA test_rbac") - ch.queryWithNoError(r, "DROP ROW POLICY test_rbac ON default.test_rbac") - ch.queryWithNoError(r, "DROP ROLE test_rbac") - ch.queryWithNoError(r, "DROP USER test_rbac") - ch.queryWithNoError(r, "DROP TABLE IF EXISTS default.test_rbac") - ch.chbackend.Close() - -} - -// TestDoRestoreConfigs - require direct access to `/etc/clickhouse-backup/`, so executed inside `clickhouse` container -// need clickhouse-server restart, no parallel -func TestDoRestoreConfigs(t *testing.T) { - if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "1.1.54391") < 0 { - t.Skipf("Test skipped, users.d is not available for %s version", os.Getenv("CLICKHOUSE_VERSION")) + if isParallel { + t.Logf("%s run in parallel mode project=%s", t.Name(), env.ProjectName) + } else { + t.Logf("%s run in sequence mode project=%s", t.Name(), env.ProjectName) } - ch := &TestClickHouse{} - r := require.New(t) - ch.connectWithWait(r, 0*time.Millisecond, 1*time.Second) - ch.queryWithNoError(r, "DROP TABLE IF EXISTS default.test_configs") - ch.queryWithNoError(r, "CREATE TABLE default.test_rbac (v UInt64) ENGINE=MergeTree() ORDER BY tuple()") - - r.NoError(dockerExec("clickhouse", "bash", "-ce", "echo '1' > /etc/clickhouse-server/users.d/test_config.xml")) - - r.NoError(dockerExec("clickhouse", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "--configs", "--configs-only", "test_configs_backup")) - ch.queryWithNoError(r, "DROP TABLE IF EXISTS default.test_configs") - r.NoError(dockerExec("clickhouse", "bash", "-xec", "CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml S3_COMPRESSION_FORMAT=none ALLOW_EMPTY_BACKUPS=1 clickhouse-backup upload test_configs_backup")) - r.NoError(dockerExec("clickhouse", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_configs_backup")) - ch.queryWithNoError(r, "SYSTEM RELOAD CONFIG") - ch.chbackend.Close() - ch.connectWithWait(r, 1*time.Second, 1*time.Second) - selectEmptyResultForAggQuery := "SELECT value FROM system.settings WHERE name='empty_result_for_aggregation_by_empty_set'" - var settings string - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&settings, selectEmptyResultForAggQuery)) - if settings != "1" { - r.NoError(dockerExec("clickhouse", "grep", "empty_result_for_aggregation_by_empty_set", "-r", "/var/lib/clickhouse/preprocessed_configs/")) + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "1.1.54394") <= 0 { + r := require.New(&testing.T{}) + env.InstallDebIfNotExists(r, "clickhouse-backup", "ca-certificates", "curl") + env.DockerExecNoError(r, "clickhouse-backup", "update-ca-certificates") } - r.Equal("1", settings, "expect empty_result_for_aggregation_by_empty_set=1") - - r.NoError(dockerExec("clickhouse", "rm", "-rfv", "/etc/clickhouse-server/users.d/test_config.xml")) - r.NoError(dockerExec("clickhouse", "bash", "-xec", "CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml ALLOW_EMPTY_BACKUPS=1 clickhouse-backup download test_configs_backup")) - - r.NoError(ch.chbackend.Query("SYSTEM RELOAD CONFIG")) - ch.chbackend.Close() - ch.connectWithWait(r, 1*time.Second, 1*time.Second) - - settings = "" - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&settings, "SELECT value FROM system.settings WHERE name='empty_result_for_aggregation_by_empty_set'")) - r.Equal("0", settings, "expect empty_result_for_aggregation_by_empty_set=0") - - r.NoError(dockerExec("clickhouse", "bash", "-xec", "CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml CLICKHOUSE_RESTART_COMMAND='sql:SYSTEM RELOAD CONFIG' clickhouse-backup restore --rm --configs --configs-only test_configs_backup")) + return env, r +} - ch.chbackend.Close() - ch.connectWithWait(r, 1*time.Second, 1*time.Second) +func (env *TestEnvironment) Cleanup(t *testing.T, r *require.Assertions) { + env.ch.Close() - settings = "" - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&settings, "SELECT value FROM system.settings WHERE name='empty_result_for_aggregation_by_empty_set'")) - r.Equal("1", settings, "expect empty_result_for_aggregation_by_empty_set=1") + if t.Name() == "TestIntegrationS3" || t.Name() == "TestIntegrationEmbedded" { + env.DockerExecNoError(r, "minio", "rm", "-rf", "/bitnami/minio/data/clickhouse/disk_s3") + } - isTestConfigsTablePresent := 0 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&isTestConfigsTablePresent, "SELECT count() FROM system.tables WHERE database='default' AND name='test_configs' SETTINGS empty_result_for_aggregation_by_empty_set=1")) - r.Equal(0, isTestConfigsTablePresent, "expect default.test_configs is not present") + if t.Name() == "TestRBAC" || t.Name() == "TestConfigs" || t.Name() == "TestIntegrationEmbedded" { + env.DockerExecNoError(r, "minio", "rm", "-rf", "/bitnami/minio/data/clickhouse/backups_s3") + } + if t.Name() == "TestIntegrationCustomRsync" { + env.DockerExecNoError(r, "sshd", "rm", "-rf", "/root/rsync_backups") + } + if t.Name() == "TestIntegrationCustomRestic" { + env.DockerExecNoError(r, "minio", "rm", "-rf", "/bitnami/minio/data/clickhouse/restic") + } + if t.Name() == "TestIntegrationCustomKopia" { + env.DockerExecNoError(r, "minio", "rm", "-rf", "/bitnami/minio/data/clickhouse/kopia") + } - r.NoError(dockerExec("clickhouse", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_configs_backup")) - r.NoError(dockerExec("clickhouse", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "remote", "test_configs_backup")) - r.NoError(dockerExec("clickhouse", "rm", "-rfv", "/etc/clickhouse-server/users.d/test_config.xml")) + if err := dockerPool.ReturnObject(context.Background(), env); err != nil { + t.Fatalf("dockerPool.ReturnObject error: %+v", err) + } - ch.chbackend.Close() } -// TestLongListRemote - no parallel, cause need to restart minito +// TestLongListRemote - no parallel, cause need to restart minio func TestLongListRemote(t *testing.T) { - ch := &TestClickHouse{} - r := require.New(t) - ch.connectWithWait(r, 0*time.Second, 1*time.Second) - defer ch.chbackend.Close() + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 0*time.Second, 1*time.Second, 1*time.Minute) totalCacheCount := 20 testBackupName := "test_list_remote" for i := 0; i < totalCacheCount; i++ { - r.NoError(dockerExec("clickhouse-backup", "bash", "-ce", fmt.Sprintf("CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml ALLOW_EMPTY_BACKUPS=true clickhouse-backup create_remote %s_%d", testBackupName, i))) + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", fmt.Sprintf("CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml ALLOW_EMPTY_BACKUPS=true clickhouse-backup create_remote %s_%d", testBackupName, i)) } - r.NoError(dockerExec("clickhouse-backup", "rm", "-rfv", "/tmp/.clickhouse-backup-metadata.cache.S3")) - r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker-compose", "-f", os.Getenv("COMPOSE_FILE"), "restart", "minio")) + r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker", append(env.GetDefaultComposeCommand(), "restart", "minio")...)) time.Sleep(2 * time.Second) startFirst := time.Now() - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "list", "remote")) + env.DockerExecNoError(r, "clickhouse-backup", "rm", "-rfv", "/tmp/.clickhouse-backup-metadata.cache.S3") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "list", "remote") noCacheDuration := time.Since(startFirst) - r.NoError(dockerExec("clickhouse-backup", "chmod", "-Rv", "+r", "/tmp/.clickhouse-backup-metadata.cache.S3")) + env.DockerExecNoError(r, "clickhouse-backup", "chmod", "-Rv", "+r", "/tmp/.clickhouse-backup-metadata.cache.S3") + + r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker", append(env.GetDefaultComposeCommand(), "restart", "minio")...)) + time.Sleep(2 * time.Second) startCashed := time.Now() - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "list", "remote")) - cashedDuration := time.Since(startCashed) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "list", "remote") + cachedDuration := time.Since(startCashed) - r.Greater(noCacheDuration, cashedDuration) + r.Greater(noCacheDuration, cachedDuration, "noCacheDuration=%s shall be greater cachedDuration=%s", noCacheDuration, cachedDuration) - r.NoError(dockerExec("clickhouse-backup", "rm", "-Rfv", "/tmp/.clickhouse-backup-metadata.cache.S3")) - r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker-compose", "-f", os.Getenv("COMPOSE_FILE"), "restart", "minio")) + r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker", append(env.GetDefaultComposeCommand(), "restart", "minio")...)) time.Sleep(2 * time.Second) startCacheClear := time.Now() - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "list", "remote")) + env.DockerExecNoError(r, "clickhouse-backup", "rm", "-Rfv", "/tmp/.clickhouse-backup-metadata.cache.S3") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "list", "remote") cacheClearDuration := time.Since(startCacheClear) - r.Greater(cacheClearDuration, cashedDuration) - log.Info().Msgf("noCacheDuration=%s cachedDuration=%s cacheClearDuration=%s", noCacheDuration.String(), cashedDuration.String(), cacheClearDuration.String()) + r.Greater(cacheClearDuration, cachedDuration, "cacheClearDuration=%s shall be greater cachedDuration=%s", cacheClearDuration.String(), cachedDuration.String()) + log.Debug().Msgf("noCacheDuration=%s cachedDuration=%s cacheClearDuration=%s", noCacheDuration.String(), cachedDuration.String(), cacheClearDuration.String()) testListRemoteAllBackups := make([]string, totalCacheCount) for i := 0; i < totalCacheCount; i++ { testListRemoteAllBackups[i] = fmt.Sprintf("%s_%d", testBackupName, i) } - fullCleanup(t, r, ch, testListRemoteAllBackups, []string{"remote", "local"}, []string{}, true, true, "config-s3.yml") + fullCleanup(t, r, env, testListRemoteAllBackups, []string{"remote", "local"}, []string{}, true, true, "config-s3.yml") + env.Cleanup(t, r) } +func TestIntegrationAzure(t *testing.T) { + if isTestShouldSkip("AZURE_TESTS") { + t.Skip("Skipping Azure integration tests...") + return + } + env, r := NewTestEnvironment(t) + env.runMainIntegrationScenario(t, "AZBLOB", "config-azblob.yml") + env.Cleanup(t, r) +} + +func TestIntegrationGCSWithCustomEndpoint(t *testing.T) { + if isTestShouldSkip("GCS_TESTS") { + t.Skip("Skipping GCS_EMULATOR integration tests...") + return + } + env, r := NewTestEnvironment(t) + env.runMainIntegrationScenario(t, "GCS_EMULATOR", "config-gcs-custom-endpoint.yml") + env.Cleanup(t, r) +} + +func TestIntegrationSFTPAuthKey(t *testing.T) { + env, r := NewTestEnvironment(t) + env.uploadSSHKeys(r, "clickhouse-backup") + env.runMainIntegrationScenario(t, "SFTP", "config-sftp-auth-key.yaml") + env.Cleanup(t, r) +} + +func TestIntegrationSFTPAuthPassword(t *testing.T) { + env, r := NewTestEnvironment(t) + env.runMainIntegrationScenario(t, "SFTP", "config-sftp-auth-password.yaml") + env.Cleanup(t, r) +} + +func TestIntegrationFTP(t *testing.T) { + env, r := NewTestEnvironment(t) + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") >= 1 { + env.runMainIntegrationScenario(t, "FTP", "config-ftp.yaml") + } else { + env.runMainIntegrationScenario(t, "FTP", "config-ftp-old.yaml") + } + env.Cleanup(t, r) +} + +func TestIntegrationS3Glacier(t *testing.T) { + if isTestShouldSkip("GLACIER_TESTS") { + t.Skip("Skipping GLACIER integration tests...") + return + } + env, r := NewTestEnvironment(t) + r.NoError(env.DockerCP("config-s3-glacier.yml", "clickhouse-backup:/etc/clickhouse-backup/config.yml.s3glacier-template")) + env.InstallDebIfNotExists(r, "clickhouse-backup", "curl", "gettext-base", "bsdmainutils", "dnsutils", "git", "ca-certificates") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "cat /etc/clickhouse-backup/config.yml.s3glacier-template | envsubst > /etc/clickhouse-backup/config-s3-glacier.yml") + dockerExecTimeout = 60 * time.Minute + env.runMainIntegrationScenario(t, "GLACIER", "config-s3-glacier.yml") + dockerExecTimeout = 3 * time.Minute + env.Cleanup(t, r) +} + +func TestIntegrationS3(t *testing.T) { + env, r := NewTestEnvironment(t) + env.checkObjectStorageIsEmpty(t, r, "S3") + env.runMainIntegrationScenario(t, "S3", "config-s3.yml") + env.Cleanup(t, r) +} + +func TestIntegrationGCS(t *testing.T) { + if isTestShouldSkip("GCS_TESTS") { + t.Skip("Skipping GCS integration tests...") + return + } + env, r := NewTestEnvironment(t) + env.runMainIntegrationScenario(t, "GCS", "config-gcs.yml") + env.Cleanup(t, r) +} + +func TestIntegrationEmbedded(t *testing.T) { + //t.Skipf("Test skipped, wait 23.8, RESTORE Ordinary table and RESTORE MATERIALIZED VIEW and {uuid} not works for %s version, look https://github.com/ClickHouse/ClickHouse/issues/43971 and https://github.com/ClickHouse/ClickHouse/issues/42709", os.Getenv("CLICKHOUSE_VERSION")) + //dependencies restore https://github.com/ClickHouse/ClickHouse/issues/39416, fixed in 23.3 + version := os.Getenv("CLICKHOUSE_VERSION") + if compareVersion(version, "23.3") < 0 { + t.Skipf("Test skipped, BACKUP/RESTORE not production ready for %s version", version) + } + env, r := NewTestEnvironment(t) + + //CUSTOM backup creates folder in each disk, need to clear + env.DockerExecNoError(r, "clickhouse", "rm", "-rfv", "/var/lib/clickhouse/disks/backups_s3/backup/") + env.runMainIntegrationScenario(t, "EMBEDDED_S3", "config-s3-embedded.yml") + + //@TODO think about how to implements embedded backup for s3_plain disks + //env.DockerExecNoError(r, "clickhouse", "rm", "-rf", "/var/lib/clickhouse/disks/backups_s3_plain/backup/") + //runMainIntegrationScenario(t, "EMBEDDED_S3_PLAIN", "config-s3-plain-embedded.yml") + + t.Log("@TODO clickhouse-server don't close connection properly after FIN from azurite during BACKUP/RESTORE https://github.com/ClickHouse/ClickHouse/issues/60447, https://github.com/Azure/Azurite/issues/2053") + //env.DockerExecNoError(r, "azure", "apk", "add", "tcpdump") + //r.NoError(env.DockerExecBackground("azure", "tcpdump", "-i", "any", "-w", "/tmp/azurite_http.pcap", "port", "10000")) + ////CUSTOM backup create folder in each disk + //env.DockerExecNoError(r, "clickhouse", "rm", "-rf", "/var/lib/clickhouse/disks/backups_azure/backup/") + //if compareVersion(version, "24.2") >= 0 { + // env.runMainIntegrationScenario(t, "EMBEDDED_AZURE_URL", "config-azblob-embedded-url.yml") + //} + //env.runMainIntegrationScenario(t, "EMBEDDED_AZURE", "config-azblob-embedded.yml") + //env.DockerExecNoError(r, "azure", "pkill", "tcpdump") + //r.NoError(env.DockerCP("azure:/tmp/azurite_http.pcap", "./azurite_http.pcap")) + + if compareVersion(version, "23.8") >= 0 { + //CUSTOM backup creates folder in each disk, need to clear + env.DockerExecNoError(r, "clickhouse", "rm", "-rfv", "/var/lib/clickhouse/disks/backups_local/backup/") + env.runMainIntegrationScenario(t, "EMBEDDED_LOCAL", "config-s3-embedded-local.yml") + } + if compareVersion(version, "24.3") >= 0 { + //@todo think about named collections to avoid show credentials in logs look to https://github.com/fsouza/fake-gcs-server/issues/1330, https://github.com/fsouza/fake-gcs-server/pull/1164 + env.InstallDebIfNotExists(r, "clickhouse-backup", "ca-certificates", "gettext-base") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "cat /etc/clickhouse-backup/config-gcs-embedded-url.yml.template | envsubst > /etc/clickhouse-backup/config-gcs-embedded-url.yml") + env.runMainIntegrationScenario(t, "EMBEDDED_GCS_URL", "config-gcs-embedded-url.yml") + env.runMainIntegrationScenario(t, "EMBEDDED_S3_URL", "config-s3-embedded-url.yml") + } + env.Cleanup(t, r) +} + + +func TestIntegrationCustomKopia(t *testing.T) { + env, r := NewTestEnvironment(t) + env.InstallDebIfNotExists(r, "clickhouse-backup", "ca-certificates", "curl") + env.DockerExecNoError(r, "clickhouse-backup", "update-ca-certificates") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xce", "command -v yq || curl -sL \"https://github.com/mikefarah/yq/releases/latest/download/yq_linux_$(dpkg --print-architecture)\" -o /usr/bin/yq && chmod +x /usr/bin/yq") + env.InstallDebIfNotExists(r, "clickhouse-backup", "jq", "bzip2", "pgp", "git") + + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", "curl -sfL https://kopia.io/signing-key | gpg --dearmor -o /usr/share/keyrings/kopia-keyring.gpg") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", "echo 'deb [signed-by=/usr/share/keyrings/kopia-keyring.gpg] https://packages.kopia.io/apt/ stable main' > /etc/apt/sources.list.d/kopia.list") + env.InstallDebIfNotExists(r, "clickhouse-backup", "kopia", "xxd", "bsdmainutils", "parallel") + + env.runIntegrationCustom(t, r, "kopia") + env.Cleanup(t, r) +} + +func TestIntegrationCustomRestic(t *testing.T) { + env, r := NewTestEnvironment(t) + env.InstallDebIfNotExists(r, "clickhouse-backup", "ca-certificates", "curl") + env.DockerExecNoError(r, "clickhouse-backup", "update-ca-certificates") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xce", "command -v yq || curl -sL \"https://github.com/mikefarah/yq/releases/latest/download/yq_linux_$(dpkg --print-architecture)\" -o /usr/bin/yq && chmod +x /usr/bin/yq") + env.InstallDebIfNotExists(r, "clickhouse-backup", "jq", "bzip2", "pgp", "git") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "command -v restic || RELEASE_TAG=$(curl -H 'Accept: application/json' -sL https://github.com/restic/restic/releases/latest | jq -c -r -M '.tag_name'); RELEASE=$(echo ${RELEASE_TAG} | sed -e 's/v//'); curl -sfL \"https://github.com/restic/restic/releases/download/${RELEASE_TAG}/restic_${RELEASE}_linux_amd64.bz2\" | bzip2 -d > /bin/restic; chmod +x /bin/restic") + env.runIntegrationCustom(t, r, "restic") + env.Cleanup(t, r) +} + +func TestIntegrationCustomRsync(t *testing.T) { + env, r := NewTestEnvironment(t) + env.uploadSSHKeys(r, "clickhouse-backup") + env.InstallDebIfNotExists(r, "clickhouse-backup", "ca-certificates", "curl") + env.DockerExecNoError(r, "clickhouse-backup", "update-ca-certificates") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xce", "command -v yq || curl -sL \"https://github.com/mikefarah/yq/releases/latest/download/yq_linux_$(dpkg --print-architecture)\" -o /usr/bin/yq && chmod +x /usr/bin/yq") + env.InstallDebIfNotExists(r, "clickhouse-backup", "jq", "openssh-client", "rsync") + env.runIntegrationCustom(t, r, "rsync") + env.Cleanup(t, r) +} + +func (env *TestEnvironment) runIntegrationCustom(t *testing.T, r *require.Assertions, customType string) { + env.DockerExecNoError(r, "clickhouse-backup", "mkdir", "-pv", "/custom/"+customType) + r.NoError(env.DockerCP("./"+customType+"/", "clickhouse-backup:/custom/")) + env.runMainIntegrationScenario(t, "CUSTOM", "config-custom-"+customType+".yml") +} + +// TestS3NoDeletePermission - no parallel +func TestS3NoDeletePermission(t *testing.T) { + if isTestShouldSkip("RUN_ADVANCED_TESTS") { + t.Skip("Skipping Advanced integration tests...") + return + } + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 500*time.Millisecond, 1*time.Second, 1*time.Minute) + + env.DockerExecNoError(r, "minio", "/bin/minio_nodelete.sh") + r.NoError(env.DockerCP("config-s3-nodelete.yml", "clickhouse-backup:/etc/clickhouse-backup/config.yml")) + + generateTestData(t, r, env, "S3", defaultTestData) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "create_remote", "no_delete_backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "delete", "local", "no_delete_backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "restore_remote", "no_delete_backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "delete", "local", "no_delete_backup") + r.Error(env.DockerExec("clickhouse-backup", "clickhouse-backup", "delete", "remote", "no_delete_backup")) + databaseList := []string{dbNameOrdinary, dbNameAtomic, dbNameMySQL, dbNamePostgreSQL, Issue331Atomic, Issue331Ordinary} + dropDatabasesFromTestDataDataSet(t, r, env, databaseList) + r.NoError(env.DockerCP("config-s3.yml", "clickhouse-backup:/etc/clickhouse-backup/config.yml")) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "delete", "remote", "no_delete_backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "list", "remote") + env.checkObjectStorageIsEmpty(t, r, "S3") + env.Cleanup(t, r) +} + +// TestRBAC need clickhouse-server restart, no parallel +func TestRBAC(t *testing.T) { + chVersion := os.Getenv("CLICKHOUSE_VERSION") + if compareVersion(chVersion, "20.4") < 0 { + t.Skipf("Test skipped, RBAC not available for %s version", os.Getenv("CLICKHOUSE_VERSION")) + } + env, r := NewTestEnvironment(t) + testRBACScenario := func(config string) { + env.connectWithWait(r, 1*time.Second, 1*time.Second, 1*time.Minute) + + env.queryWithNoError(r, "DROP TABLE IF EXISTS default.test_rbac") + env.queryWithNoError(r, "CREATE TABLE default.test_rbac (v UInt64) ENGINE=MergeTree() ORDER BY tuple()") + env.queryWithNoError(r, "DROP SETTINGS PROFILE IF EXISTS `test.rbac-name`") + env.queryWithNoError(r, "DROP QUOTA IF EXISTS `test.rbac-name`") + env.queryWithNoError(r, "DROP ROW POLICY IF EXISTS `test.rbac-name` ON default.test_rbac") + env.queryWithNoError(r, "DROP ROLE IF EXISTS `test.rbac-name`") + env.queryWithNoError(r, "DROP USER IF EXISTS `test.rbac-name`") + + createRBACObjects := func(drop bool) { + if drop { + log.Debug().Msg("drop all RBAC related objects") + env.queryWithNoError(r, "DROP SETTINGS PROFILE `test.rbac-name`") + env.queryWithNoError(r, "DROP QUOTA `test.rbac-name`") + env.queryWithNoError(r, "DROP ROW POLICY `test.rbac-name` ON default.test_rbac") + env.queryWithNoError(r, "DROP ROLE `test.rbac-name`") + env.queryWithNoError(r, "DROP USER `test.rbac-name`") + } + log.Debug().Msg("create RBAC related objects") + env.queryWithNoError(r, "CREATE SETTINGS PROFILE `test.rbac-name` SETTINGS max_execution_time=60") + env.queryWithNoError(r, "CREATE ROLE `test.rbac-name` SETTINGS PROFILE `test.rbac-name`") + env.queryWithNoError(r, "CREATE USER `test.rbac-name` IDENTIFIED BY 'test_rbac_password' DEFAULT ROLE `test.rbac-name`") + env.queryWithNoError(r, "CREATE QUOTA `test.rbac-name` KEYED BY user_name FOR INTERVAL 1 hour NO LIMITS TO `test.rbac-name`") + env.queryWithNoError(r, "CREATE ROW POLICY `test.rbac-name` ON default.test_rbac USING 1=1 AS RESTRICTIVE TO `test.rbac-name`") + } + createRBACObjects(false) + + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", config, "create", "--rbac", "--rbac-only", "--env", "S3_COMPRESSION_FORMAT=zstd", "test_rbac_backup") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "ALLOW_EMPTY_BACKUPS=1 CLICKHOUSE_BACKUP_CONFIG="+config+" clickhouse-backup upload test_rbac_backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", config, "delete", "local", "test_rbac_backup") + env.DockerExecNoError(r, "clickhouse", "ls", "-lah", "/var/lib/clickhouse/access") + + log.Debug().Msg("create conflicted RBAC objects") + createRBACObjects(true) + + env.DockerExecNoError(r, "clickhouse", "ls", "-lah", "/var/lib/clickhouse/access") + + log.Debug().Msg("download+restore RBAC") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "ALLOW_EMPTY_BACKUPS=1 CLICKHOUSE_BACKUP_CONFIG="+config+" clickhouse-backup download test_rbac_backup") + + out, err := env.DockerExecOut("clickhouse-backup", "bash", "-xec", "ALLOW_EMPTY_BACKUPS=1 clickhouse-backup -c "+config+" restore --rm --rbac test_rbac_backup") + log.Debug().Msg(out) + r.Contains(out, "RBAC successfully restored") + r.NoError(err) + + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-xec", "ALLOW_EMPTY_BACKUPS=1 clickhouse-backup -c "+config+" restore --rm --rbac-only test_rbac_backup") + log.Debug().Msg(out) + r.Contains(out, "RBAC successfully restored") + r.NoError(err) + env.DockerExecNoError(r, "clickhouse", "ls", "-lah", "/var/lib/clickhouse/access") + + env.ch.Close() + // r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, append(env.GetDefaultComposeCommand(), "restart", "clickhouse"))) + env.connectWithWait(r, 2*time.Second, 2*time.Second, 1*time.Minute) + + env.DockerExecNoError(r, "clickhouse", "ls", "-lah", "/var/lib/clickhouse/access") + + rbacTypes := map[string]string{ + "PROFILES": "test.rbac-name", + "QUOTAS": "test.rbac-name", + "POLICIES": "`test.rbac-name` ON default.test_rbac", + "ROLES": "test.rbac-name", + "USERS": "test.rbac-name", + } + for rbacType, expectedValue := range rbacTypes { + var rbacRows []struct { + Name string `ch:"name"` + } + err := env.ch.Select(&rbacRows, fmt.Sprintf("SHOW %s", rbacType)) + r.NoError(err) + found := false + for _, row := range rbacRows { + log.Debug().Msgf("rbacType=%s expectedValue=%s row.Name=%s", rbacType, expectedValue, row.Name) + if expectedValue == row.Name { + found = true + break + } + } + if !found { + //env.DockerExecNoError(r, "clickhouse", "cat", "/var/log/clickhouse-server/clickhouse-server.log") + r.Failf("wrong RBAC", "SHOW %s, %#v doesn't contain %#v", rbacType, rbacRows, expectedValue) + } + } + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", config, "delete", "local", "test_rbac_backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", config, "delete", "remote", "test_rbac_backup") + + env.queryWithNoError(r, "DROP SETTINGS PROFILE `test.rbac-name`") + env.queryWithNoError(r, "DROP QUOTA `test.rbac-name`") + env.queryWithNoError(r, "DROP ROW POLICY `test.rbac-name` ON default.test_rbac") + env.queryWithNoError(r, "DROP ROLE `test.rbac-name`") + env.queryWithNoError(r, "DROP USER `test.rbac-name`") + env.queryWithNoError(r, "DROP TABLE IF EXISTS default.test_rbac") + env.ch.Close() + } + testRBACScenario("/etc/clickhouse-backup/config-s3.yml") + if compareVersion(chVersion, "24.1") >= 0 { + testRBACScenario("/etc/clickhouse-backup/config-s3-embedded.yml") + testRBACScenario("/etc/clickhouse-backup/config-s3-embedded-url.yml") + testRBACScenario("/etc/clickhouse-backup/config-azblob-embedded.yml") + } + if compareVersion(chVersion, "24.2") >= 0 { + testRBACScenario("/etc/clickhouse-backup/config-azblob-embedded-url.yml") + } + env.Cleanup(t, r) +} + +// TestConfigs - require direct access to `/etc/clickhouse-backup/`, so executed inside `clickhouse` container +// need clickhouse-server restart, no parallel +func TestConfigs(t *testing.T) { + env, r := NewTestEnvironment(t) + testConfigsScenario := func(config string) { + env.connectWithWait(r, 0*time.Millisecond, 1*time.Second, 1*time.Minute) + env.queryWithNoError(r, "DROP TABLE IF EXISTS default.test_configs") + env.queryWithNoError(r, "CREATE TABLE default.test_configs (v UInt64) ENGINE=MergeTree() ORDER BY tuple()") + + env.DockerExecNoError(r, "clickhouse", "bash", "-ce", "echo '1' > /etc/clickhouse-server/users.d/test_config.xml") + + env.DockerExecNoError(r, "clickhouse", "clickhouse-backup", "-c", config, "create", "--configs", "--configs-only", "test_configs_backup") + env.queryWithNoError(r, "DROP TABLE IF EXISTS default.test_configs") + compression := "" + if !strings.Contains(config, "embedded") { + compression = "--env AZBLOB_COMPRESSION_FORMAT=zstd --env S3_COMPRESSION_FORMAT=zstd" + } + env.DockerExecNoError(r, "clickhouse", "bash", "-xec", "clickhouse-backup upload "+compression+" --env CLICKHOUSE_BACKUP_CONFIG="+config+" --env S3_COMPRESSION_FORMAT=none --env ALLOW_EMPTY_BACKUPS=1 test_configs_backup") + env.DockerExecNoError(r, "clickhouse", "clickhouse-backup", "-c", config, "delete", "local", "test_configs_backup") + + env.queryWithNoError(r, "SYSTEM RELOAD CONFIG") + env.ch.Close() + env.connectWithWait(r, 1*time.Second, 1*time.Second, 1*time.Minute) + selectEmptyResultForAggQuery := "SELECT value FROM system.settings WHERE name='empty_result_for_aggregation_by_empty_set'" + var settings string + r.NoError(env.ch.SelectSingleRowNoCtx(&settings, selectEmptyResultForAggQuery)) + if settings != "1" { + env.DockerExecNoError(r, "clickhouse", "grep", "empty_result_for_aggregation_by_empty_set", "-r", "/var/lib/clickhouse/preprocessed_configs/") + } + r.Equal("1", settings, "expect empty_result_for_aggregation_by_empty_set=1") + + env.DockerExecNoError(r, "clickhouse", "rm", "-rfv", "/etc/clickhouse-server/users.d/test_config.xml") + env.DockerExecNoError(r, "clickhouse", "bash", "-xec", "CLICKHOUSE_BACKUP_CONFIG="+config+" ALLOW_EMPTY_BACKUPS=1 clickhouse-backup download test_configs_backup") + + r.NoError(env.ch.Query("SYSTEM RELOAD CONFIG")) + env.ch.Close() + env.connectWithWait(r, 1*time.Second, 1*time.Second, 1*time.Minute) + + settings = "" + r.NoError(env.ch.SelectSingleRowNoCtx(&settings, "SELECT value FROM system.settings WHERE name='empty_result_for_aggregation_by_empty_set'")) + r.Equal("0", settings, "expect empty_result_for_aggregation_by_empty_set=0") + + env.DockerExecNoError(r, "clickhouse", "bash", "-xec", "CLICKHOUSE_BACKUP_CONFIG="+config+" CLICKHOUSE_RESTART_COMMAND='sql:SYSTEM RELOAD CONFIG' clickhouse-backup restore --rm --configs --configs-only test_configs_backup") + + env.ch.Close() + env.connectWithWait(r, 1*time.Second, 1*time.Second, 1*time.Second) + + settings = "" + r.NoError(env.ch.SelectSingleRowNoCtx(&settings, "SELECT value FROM system.settings WHERE name='empty_result_for_aggregation_by_empty_set'")) + r.Equal("1", settings, "expect empty_result_for_aggregation_by_empty_set=1") + + isTestConfigsTablePresent := 0 + r.NoError(env.ch.SelectSingleRowNoCtx(&isTestConfigsTablePresent, "SELECT count() FROM system.tables WHERE database='default' AND name='test_configs' SETTINGS empty_result_for_aggregation_by_empty_set=1")) + r.Equal(0, isTestConfigsTablePresent, "expect default.test_configs is not present") + + env.DockerExecNoError(r, "clickhouse", "clickhouse-backup", "-c", config, "delete", "local", "test_configs_backup") + env.DockerExecNoError(r, "clickhouse", "clickhouse-backup", "-c", config, "delete", "remote", "test_configs_backup") + env.DockerExecNoError(r, "clickhouse", "rm", "-rfv", "/etc/clickhouse-server/users.d/test_config.xml") + + env.ch.Close() + } + testConfigsScenario("/etc/clickhouse-backup/config-s3.yml") + chVersion := os.Getenv("CLICKHOUSE_VERSION") + if compareVersion(chVersion, "24.1") >= 0 { + testConfigsScenario("/etc/clickhouse-backup/config-s3-embedded.yml") + testConfigsScenario("/etc/clickhouse-backup/config-s3-embedded-url.yml") + testConfigsScenario("/etc/clickhouse-backup/config-azblob-embedded.yml") + } + if compareVersion(chVersion, "24.2") >= 0 { + testConfigsScenario("/etc/clickhouse-backup/config-azblob-embedded-url.yml") + } + env.Cleanup(t, r) +} + +const apiBackupNumber = 5 + func TestServerAPI(t *testing.T) { - ch := &TestClickHouse{} - r := require.New(t) - ch.connectWithWait(r, 0*time.Second, 10*time.Second) - defer func() { - ch.chbackend.Close() - }() - r.NoError(dockerCP("config-s3.yml", "clickhouse-backup:/etc/clickhouse-backup/config.yml")) + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 0*time.Second, 1*time.Second, 1*time.Minute) + r.NoError(env.DockerCP("config-s3.yml", "clickhouse-backup:/etc/clickhouse-backup/config.yml")) fieldTypes := []string{"UInt64", "String", "Int"} - installDebIfNotExists(r, "clickhouse-backup", "curl") + env.InstallDebIfNotExists(r, "clickhouse-backup", "curl", "jq") maxTables := 10 minFields := 10 randFields := 10 - fillDatabaseForAPIServer(maxTables, minFields, randFields, ch, r, fieldTypes) + fillDatabaseForAPIServer(maxTables, minFields, randFields, env, r, fieldTypes) - log.Info().Msg("Run `clickhouse-backup server --watch` in background") - r.NoError(dockerExec("-d", "clickhouse-backup", "bash", "-ce", "clickhouse-backup server --watch &>>/tmp/clickhouse-backup-server.log")) + log.Debug().Msg("Run `clickhouse-backup server --watch` in background") + env.DockerExecBackgroundNoError(r,"clickhouse-backup", "bash", "-ce", "clickhouse-backup server --watch &>>/tmp/clickhouse-backup-server.log") time.Sleep(1 * time.Second) - testAPIBackupCreate(r) + testAPIBackupVersion(r, env) + + testAPIBackupCreate(r, env) - testAPIBackupTables(r) + testAPIBackupTables(r, env) - log.Info().Msg("Check /backup/actions") - ch.queryWithNoError(r, "SELECT count() FROM system.backup_actions") + testAPIBackupUpload(r, env) - testAPIBackupUpload(r) + testAPIBackupTablesRemote(r, env) - testAPIBackupList(t, r) + log.Debug().Msg("Check /backup/actions") + env.queryWithNoError(r, "SELECT count() FROM system.backup_actions") - testAPIDeleteLocalDownloadRestore(r) + testAPIBackupList(t, r, env) - testAPIMetrics(r, ch) + testAPIDeleteLocalDownloadRestore(r, env) - testAPIWatchAndKill(r, ch) + testAPIMetrics(r, env) - testAPIBackupActions(r, ch) + testAPIWatchAndKill(r, env) - testAPIRestart(r, ch) + testAPIBackupActions(r, env) - testAPIBackupDelete(r) + testAPIRestart(r, env) - r.NoError(dockerExec("clickhouse-backup", "pkill", "-n", "-f", "clickhouse-backup")) - r.NoError(ch.dropDatabase("long_schema")) + testAPIBackupDelete(r, env) + + testAPIBackupClean(r, env) + + env.DockerExecNoError(r, "clickhouse-backup", "pkill", "-n", "-f", "clickhouse-backup") + r.NoError(env.dropDatabase("long_schema")) + env.Cleanup(t, r) } -func testAPIRestart(r *require.Assertions, ch *TestClickHouse) { - out, err := dockerExecOut("clickhouse-backup", "bash", "-ce", "curl -sfL -XPOST 'http://localhost:7171/restart'") +func testAPIRestart(r *require.Assertions, env *TestEnvironment) { + out, err := env.DockerExecOut("clickhouse-backup", "bash", "-ce", "curl -sfL -XPOST 'http://localhost:7171/restart'") log.Debug().Msg(out) r.NoError(err) r.Contains(out, "acknowledged") @@ -696,13 +994,13 @@ func testAPIRestart(r *require.Assertions, ch *TestClickHouse) { time.Sleep(6 * time.Second) var inProgressActions uint64 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&inProgressActions, "SELECT count() FROM system.backup_actions WHERE status!=?", status.CancelStatus)) + r.NoError(env.ch.SelectSingleRowNoCtx(&inProgressActions, "SELECT count() FROM system.backup_actions WHERE status!=?", status.CancelStatus)) r.Equal(uint64(0), inProgressActions) } -func runClickHouseClientInsertSystemBackupActions(r *require.Assertions, ch *TestClickHouse, commands []string, needWait bool) { +func runClickHouseClientInsertSystemBackupActions(r *require.Assertions, env *TestEnvironment, commands []string, needWait bool) { sql := "INSERT INTO system.backup_actions(command) " + "VALUES ('" + strings.Join(commands, "'),('") + "')" - out, err := dockerExecOut("clickhouse", "bash", "-ce", fmt.Sprintf("clickhouse client --echo -mn -q \"%s\"", sql)) + out, err := env.DockerExecOut("clickhouse", "bash", "-ce", fmt.Sprintf("clickhouse client --echo -mn -q \"%s\"", sql)) log.Debug().Msg(out) r.NoError(err) if needWait { @@ -710,7 +1008,7 @@ func runClickHouseClientInsertSystemBackupActions(r *require.Assertions, ch *Tes for { time.Sleep(500 * time.Millisecond) var commandStatus string - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&commandStatus, "SELECT status FROM system.backup_actions WHERE command=?", command)) + r.NoError(env.ch.SelectSingleRowNoCtx(&commandStatus, "SELECT status FROM system.backup_actions WHERE command=?", command)) if commandStatus != status.InProgressStatus { break } @@ -718,30 +1016,31 @@ func runClickHouseClientInsertSystemBackupActions(r *require.Assertions, ch *Tes } } } -func testAPIBackupActions(r *require.Assertions, ch *TestClickHouse) { - runClickHouseClientInsertSystemBackupActions(r, ch, []string{"create_remote actions_backup1"}, true) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{"delete local actions_backup1", "restore_remote --rm actions_backup1"}, true) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{"delete local actions_backup1", "delete remote actions_backup1"}, false) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{"create actions_backup2"}, true) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{"upload actions_backup2"}, true) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{"delete local actions_backup2"}, false) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{"download actions_backup2"}, true) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{"restore --rm actions_backup2"}, true) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{"delete local actions_backup2", "delete remote actions_backup2"}, false) +func testAPIBackupActions(r *require.Assertions, env *TestEnvironment) { + runClickHouseClientInsertSystemBackupActions(r, env, []string{"create_remote actions_backup1"}, true) + runClickHouseClientInsertSystemBackupActions(r, env, []string{"delete local actions_backup1", "restore_remote --rm actions_backup1"}, true) + runClickHouseClientInsertSystemBackupActions(r, env, []string{"delete local actions_backup1", "delete remote actions_backup1"}, false) + + runClickHouseClientInsertSystemBackupActions(r, env, []string{"create actions_backup2"}, true) + runClickHouseClientInsertSystemBackupActions(r, env, []string{"upload actions_backup2"}, true) + runClickHouseClientInsertSystemBackupActions(r, env, []string{"delete local actions_backup2"}, false) + runClickHouseClientInsertSystemBackupActions(r, env, []string{"download actions_backup2"}, true) + runClickHouseClientInsertSystemBackupActions(r, env, []string{"restore --rm actions_backup2"}, true) + runClickHouseClientInsertSystemBackupActions(r, env, []string{"delete local actions_backup2", "delete remote actions_backup2"}, false) inProgressActions := make([]struct { Command string `ch:"command"` Status string `ch:"status"` }, 0) - r.NoError(ch.chbackend.StructSelect(&inProgressActions, "SELECT command, status FROM system.backup_actions WHERE command LIKE '%actions%' AND status IN (?,?)", status.InProgressStatus, status.ErrorStatus)) + r.NoError(env.ch.StructSelect(&inProgressActions, "SELECT command, status FROM system.backup_actions WHERE command LIKE '%actions%' AND status IN (?,?)", status.InProgressStatus, status.ErrorStatus)) r.Equal(0, len(inProgressActions), "inProgressActions=%+v", inProgressActions) var actionsBackups uint64 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&actionsBackups, "SELECT count() FROM system.backup_list WHERE name LIKE 'backup_action%'")) + r.NoError(env.ch.SelectSingleRowNoCtx(&actionsBackups, "SELECT count() FROM system.backup_list WHERE name LIKE 'backup_action%'")) r.Equal(uint64(0), actionsBackups) - out, err := dockerExecOut("clickhouse-backup", "curl", "http://localhost:7171/metrics") + out, err := env.DockerExecOut("clickhouse-backup", "curl", "http://localhost:7171/metrics") r.NoError(err) r.Contains(out, "clickhouse_backup_last_create_remote_status 1") r.Contains(out, "clickhouse_backup_last_create_status 1") @@ -751,16 +1050,16 @@ func testAPIBackupActions(r *require.Assertions, ch *TestClickHouse) { r.Contains(out, "clickhouse_backup_last_restore_status 1") } -func testAPIWatchAndKill(r *require.Assertions, ch *TestClickHouse) { - log.Info().Msg("Check /backup/watch + /backup/kill") +func testAPIWatchAndKill(r *require.Assertions, env *TestEnvironment) { + log.Debug().Msg("Check /backup/watch + /backup/kill") runKillCommand := func(command string) { - out, err := dockerExecOut("clickhouse-backup", "bash", "-ce", fmt.Sprintf("curl -sfL 'http://localhost:7171/backup/kill?command=%s'", command)) + out, err := env.DockerExecOut("clickhouse-backup", "bash", "-ce", fmt.Sprintf("curl -sfL 'http://localhost:7171/backup/kill?command=%s'", command)) log.Debug().Msg(out) r.NoError(err) } checkWatchBackup := func(expectedCount uint64) { var watchBackups uint64 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&watchBackups, "SELECT count() FROM system.backup_list WHERE name LIKE 'shard%'")) + r.NoError(env.ch.SelectSingleRowNoCtx(&watchBackups, "SELECT count() FROM system.backup_list WHERE name LIKE 'shard%'")) r.Equal(expectedCount, watchBackups) } @@ -769,7 +1068,7 @@ func testAPIWatchAndKill(r *require.Assertions, ch *TestClickHouse) { Status string `ch:"status"` Command string `ch:"command"` }, 0) - r.NoError(ch.chbackend.StructSelect(&canceledCommands, "SELECT status, command FROM system.backup_actions WHERE command LIKE 'watch%'")) + r.NoError(env.ch.StructSelect(&canceledCommands, "SELECT status, command FROM system.backup_actions WHERE command LIKE 'watch%'")) r.Equal(expectedCount, len(canceledCommands)) for i := range canceledCommands { r.Equal("watch", canceledCommands[i].Command) @@ -781,65 +1080,106 @@ func testAPIWatchAndKill(r *require.Assertions, ch *TestClickHouse) { runKillCommand("watch") checkCanceledCommand(1) - out, err := dockerExecOut("clickhouse-backup", "bash", "-ce", "curl -sfL 'http://localhost:7171/backup/watch'") + out, err := env.DockerExecOut("clickhouse-backup", "bash", "-ce", "curl -sfL 'http://localhost:7171/backup/watch'") log.Debug().Msg(out) r.NoError(err) time.Sleep(7 * time.Second) - checkWatchBackup(2) + checkWatchBackup(1) runKillCommand("watch") checkCanceledCommand(2) } -func testAPIBackupDelete(r *require.Assertions) { - log.Info().Msg("Check /backup/delete/{where}/{name}") +func testAPIBackupDelete(r *require.Assertions, env *TestEnvironment) { + log.Debug().Msg("Check /backup/delete/{where}/{name}") for i := 1; i <= apiBackupNumber; i++ { - out, err := dockerExecOut("clickhouse-backup", "bash", "-ce", fmt.Sprintf("curl -sfL -XPOST 'http://localhost:7171/backup/delete/local/z_backup_%d'", i)) - log.Info().Msgf(out) + out, err := env.DockerExecOut("clickhouse-backup", "bash", "-ce", fmt.Sprintf("curl -sfL -XPOST 'http://localhost:7171/backup/delete/local/z_backup_%d'", i)) + log.Debug().Msgf(out) r.NoError(err) r.NotContains(out, "another operation is currently running") r.NotContains(out, "\"status\":\"error\"") - out, err = dockerExecOut("clickhouse-backup", "bash", "-ce", fmt.Sprintf("curl -sfL -XPOST 'http://localhost:7171/backup/delete/remote/z_backup_%d'", i)) - log.Info().Msgf(out) + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", fmt.Sprintf("curl -sfL -XPOST 'http://localhost:7171/backup/delete/remote/z_backup_%d'", i)) + log.Debug().Msgf(out) r.NoError(err) r.NotContains(out, "another operation is currently running") r.NotContains(out, "\"status\":\"error\"") } - out, err := dockerExecOut("clickhouse-backup", "curl", "http://localhost:7171/metrics") + out, err := env.DockerExecOut("clickhouse-backup", "curl", "http://localhost:7171/metrics") r.NoError(err) r.Contains(out, "clickhouse_backup_last_delete_status 1") + + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", fmt.Sprintf("curl -sfL -XGET 'http://localhost:7171/backup/list'")) + log.Debug().Msgf(out) + r.NoError(err) + scanner := bufio.NewScanner(strings.NewReader(out)) + for scanner.Scan() { + type backupJSON struct { + Name string `json:"name"` + Created string `json:"created"` + Size uint64 `json:"size,omitempty"` + Location string `json:"location"` + RequiredBackup string `json:"required"` + Desc string `json:"desc"` + } + listItem := backupJSON{} + r.NoError(json.Unmarshal(scanner.Bytes(), &listItem)) + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", fmt.Sprintf("curl -sfL -XPOST 'http://localhost:7171/backup/delete/%s/%s'", listItem.Location, listItem.Name)) + log.Debug().Msgf(out) + r.NoError(err) + } + + r.NoError(scanner.Err()) + } -func testAPIMetrics(r *require.Assertions, ch *TestClickHouse) { - log.Info().Msg("Check /metrics clickhouse_backup_last_backup_size_remote") +func testAPIBackupClean(r *require.Assertions, env *TestEnvironment) { + log.Debug().Msg("Check /backup/clean/ /backup/clean_remote_broken/ and /backup/actions fot these two commands") + out, err := env.DockerExecOut("clickhouse-backup", "bash", "-ce", fmt.Sprintf("curl -sfL -XPOST 'http://localhost:7171/backup/clean'")) + log.Debug().Msgf(out) + r.NoError(err) + r.NotContains(out, "another operation is currently running") + r.NotContains(out, "\"status\":\"error\"") + + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", fmt.Sprintf("curl -sfL -XPOST 'http://localhost:7171/backup/clean/remote_broken'")) + log.Debug().Msgf(out) + r.NoError(err) + r.NotContains(out, "another operation is currently running") + r.NotContains(out, "\"status\":\"error\"") + + runClickHouseClientInsertSystemBackupActions(r, env, []string{"clean", "clean_remote_broken"}, false) +} + +func testAPIMetrics(r *require.Assertions, env *TestEnvironment) { + log.Debug().Msg("Check /metrics clickhouse_backup_last_backup_size_remote") var lastRemoteSize int64 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&lastRemoteSize, "SELECT size FROM system.backup_list WHERE name='z_backup_5' AND location='remote'")) + r.NoError(env.ch.SelectSingleRowNoCtx(&lastRemoteSize, "SELECT size FROM system.backup_list WHERE name='z_backup_5' AND location='remote'")) var realTotalBytes uint64 if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.8") >= 0 { - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&realTotalBytes, "SELECT sum(total_bytes) FROM system.tables WHERE database='long_schema'")) + r.NoError(env.ch.SelectSingleRowNoCtx(&realTotalBytes, "SELECT sum(total_bytes) FROM system.tables WHERE database='long_schema'")) } else { - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&realTotalBytes, "SELECT sum(bytes_on_disk) FROM system.parts WHERE database='long_schema'")) + r.NoError(env.ch.SelectSingleRowNoCtx(&realTotalBytes, "SELECT sum(bytes_on_disk) FROM system.parts WHERE database='long_schema'")) } r.Greater(realTotalBytes, uint64(0)) r.Greater(uint64(lastRemoteSize), realTotalBytes) - out, err := dockerExecOut("clickhouse-backup", "curl", "-sL", "http://localhost:7171/metrics") + out, err := env.DockerExecOut("clickhouse-backup", "curl", "-sL", "http://localhost:7171/metrics") log.Debug().Msg(out) r.NoError(err) r.Contains(out, fmt.Sprintf("clickhouse_backup_last_backup_size_remote %d", lastRemoteSize)) - log.Info().Msg("Check /metrics clickhouse_backup_number_backups_*") + log.Debug().Msg("Check /metrics clickhouse_backup_number_backups_*") r.Contains(out, fmt.Sprintf("clickhouse_backup_number_backups_local %d", apiBackupNumber)) // +1 watch backup + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "list", "remote") r.Contains(out, fmt.Sprintf("clickhouse_backup_number_backups_remote %d", apiBackupNumber+1)) r.Contains(out, "clickhouse_backup_number_backups_local_expected 0") r.Contains(out, "clickhouse_backup_number_backups_remote_expected 0") } -func testAPIDeleteLocalDownloadRestore(r *require.Assertions) { - log.Info().Msg("Check /backup/delete/local/{name} + /backup/download/{name} + /backup/restore/{name}?rm=1") - out, err := dockerExecOut( +func testAPIDeleteLocalDownloadRestore(r *require.Assertions, env *TestEnvironment) { + log.Debug().Msg("Check /backup/delete/local/{name} + /backup/download/{name} + /backup/restore/{name}?rm=1") + out, err := env.DockerExecOut( "clickhouse-backup", "bash", "-xe", "-c", fmt.Sprintf("for i in {1..%d}; do date; curl -sfL -XPOST \"http://localhost:7171/backup/delete/local/z_backup_$i\"; curl -sfL -XPOST \"http://localhost:7171/backup/download/z_backup_$i\"; sleep 2; curl -sfL -XPOST \"http://localhost:7171/backup/restore/z_backup_$i?rm=1\"; sleep 8; done", apiBackupNumber), @@ -847,18 +1187,22 @@ func testAPIDeleteLocalDownloadRestore(r *require.Assertions) { log.Debug().Msg(out) r.NoError(err) r.NotContains(out, "another operation is currently running") + r.NotContains(out, "error") + + out, err = env.DockerExecOut("clickhouse-backup", "curl", "-sfL", "http://localhost:7171/backup/actions?filter=download") + r.NoError(err) r.NotContains(out, "\"status\":\"error\"") - out, err = dockerExecOut("clickhouse-backup", "curl", "http://localhost:7171/metrics") + out, err = env.DockerExecOut("clickhouse-backup", "curl", "http://localhost:7171/metrics") r.NoError(err) r.Contains(out, "clickhouse_backup_last_delete_status 1") r.Contains(out, "clickhouse_backup_last_download_status 1") r.Contains(out, "clickhouse_backup_last_restore_status 1") } -func testAPIBackupList(t *testing.T, r *require.Assertions) { - log.Info().Msg("Check /backup/list") - out, err := dockerExecOut("clickhouse-backup", "bash", "-ce", "curl -sfL 'http://localhost:7171/backup/list'") +func testAPIBackupList(t *testing.T, r *require.Assertions, env *TestEnvironment) { + log.Debug().Msg("Check /backup/list") + out, err := env.DockerExecOut("clickhouse-backup", "bash", "-ce", "curl -sfL 'http://localhost:7171/backup/list'") log.Debug().Msg(out) r.NoError(err) for i := 1; i <= apiBackupNumber; i++ { @@ -866,8 +1210,8 @@ func testAPIBackupList(t *testing.T, r *require.Assertions) { r.True(assert.Regexp(t, regexp.MustCompile(fmt.Sprintf("{\"name\":\"z_backup_%d\",\"created\":\"\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\",\"size\":\\d+,\"location\":\"remote\",\"required\":\"\",\"desc\":\"tar, regular\"}", i)), out)) } - log.Info().Msg("Check /backup/list/local") - out, err = dockerExecOut("clickhouse-backup", "bash", "-ce", "curl -sfL 'http://localhost:7171/backup/list/local'") + log.Debug().Msg("Check /backup/list/local") + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", "curl -sfL 'http://localhost:7171/backup/list/local'") log.Debug().Msg(out) r.NoError(err) for i := 1; i <= apiBackupNumber; i++ { @@ -875,8 +1219,8 @@ func testAPIBackupList(t *testing.T, r *require.Assertions) { r.True(assert.NotRegexp(t, regexp.MustCompile(fmt.Sprintf("{\"name\":\"z_backup_%d\",\"created\":\"\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\",\"size\":\\d+,\"location\":\"remote\",\"required\":\"\",\"desc\":\"tar, regular\"}", i)), out)) } - log.Info().Msg("Check /backup/list/remote") - out, err = dockerExecOut("clickhouse-backup", "bash", "-ce", "curl -sfL 'http://localhost:7171/backup/list/remote'") + log.Debug().Msg("Check /backup/list/remote") + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", "curl -sfL 'http://localhost:7171/backup/list/remote'") log.Debug().Msg(out) r.NoError(err) for i := 1; i <= apiBackupNumber; i++ { @@ -885,25 +1229,30 @@ func testAPIBackupList(t *testing.T, r *require.Assertions) { } } -func testAPIBackupUpload(r *require.Assertions) { - log.Info().Msg("Check /backup/upload") - out, err := dockerExecOut( +func testAPIBackupUpload(r *require.Assertions, env *TestEnvironment) { + log.Debug().Msg("Check /backup/upload") + out, err := env.DockerExecOut( "clickhouse-backup", "bash", "-xe", "-c", fmt.Sprintf("for i in {1..%d}; do date; curl -sfL -XPOST \"http://localhost:7171/backup/upload/z_backup_$i\"; sleep 2; done", apiBackupNumber), ) log.Debug().Msg(out) r.NoError(err) - r.NotContains(out, "\"status\":\"error\"") + r.NotContains(out, "error") r.NotContains(out, "another operation is currently running") - out, err = dockerExecOut("clickhouse-backup", "curl", "http://localhost:7171/metrics") + + out, err = env.DockerExecOut("clickhouse-backup", "curl", "-sfL", "http://localhost:7171/backup/actions?filter=upload") + r.NoError(err) + r.NotContains(out, "error") + + out, err = env.DockerExecOut("clickhouse-backup", "curl", "http://localhost:7171/metrics") r.NoError(err) r.Contains(out, "clickhouse_backup_last_upload_status 1") } -func testAPIBackupTables(r *require.Assertions) { - log.Info().Msg("Check /backup/tables") - out, err := dockerExecOut( +func testAPIBackupTables(r *require.Assertions, env *TestEnvironment) { + log.Debug().Msg("Check /backup/tables") + out, err := env.DockerExecOut( "clickhouse-backup", "bash", "-xe", "-c", "curl -sfL \"http://localhost:7171/backup/tables\"", ) @@ -917,8 +1266,8 @@ func testAPIBackupTables(r *require.Assertions) { r.NotContains(out, "INFORMATION_SCHEMA") r.NotContains(out, "information_schema") - log.Info().Msg("Check /backup/tables/all") - out, err = dockerExecOut( + log.Debug().Msg("Check /backup/tables/all") + out, err = env.DockerExecOut( "clickhouse-backup", "bash", "-xe", "-c", "curl -sfL \"http://localhost:7171/backup/tables/all\"", ) @@ -935,9 +1284,40 @@ func testAPIBackupTables(r *require.Assertions) { } } -func testAPIBackupCreate(r *require.Assertions) { - log.Info().Msg("Check /backup/create") - out, err := dockerExecOut( +func testAPIBackupTablesRemote(r *require.Assertions, env *TestEnvironment) { + + log.Debug().Msg("Check /backup/tables?remote_backup=z_backup_1") + out, err := env.DockerExecOut( + "clickhouse-backup", + "bash", "-xe", "-c", "curl -sfL \"http://localhost:7171/backup/tables?remote_backup=z_backup_1\"", + ) + log.Debug().Msg(out) + r.NoError(err) + r.Contains(out, "long_schema") + r.NotContains(out, "system") + r.NotContains(out, "Connection refused") + r.NotContains(out, "another operation is currently running") + r.NotContains(out, "\"status\":\"error\"") + r.NotContains(out, "INFORMATION_SCHEMA") + r.NotContains(out, "information_schema") + +} + +func testAPIBackupVersion(r *require.Assertions, env *TestEnvironment) { + log.Debug().Msg("Check /backup/version") + cliVersion, err := env.DockerExecOut("clickhouse-backup", "bash", "-ce", "clickhouse-backup --version 2>/dev/null --version | grep 'Version' | cut -d ':' -f 2 | xargs") + r.NoError(err) + apiVersion, err := env.DockerExecOut("clickhouse-backup", "bash", "-ce", "curl -sL http://localhost:7171/backup/version | jq -r .version") + r.NoError(err) + r.Equal(cliVersion, apiVersion) + tablesVersion, err := env.DockerExecOut("clickhouse", "bash", "-ce", "clickhouse client -q 'SELECT * FROM system.backup_version FORMAT TSVRaw'") + r.NoError(err) + r.Equal(cliVersion, tablesVersion) +} + +func testAPIBackupCreate(r *require.Assertions, env *TestEnvironment) { + log.Debug().Msg("Check /backup/create") + out, err := env.DockerExecOut( "clickhouse-backup", "bash", "-xe", "-c", fmt.Sprintf("sleep 3; for i in {1..%d}; do date; curl -sfL -XPOST \"http://localhost:7171/backup/create?table=long_schema.*&name=z_backup_$i\"; sleep 1.5; done", apiBackupNumber), @@ -947,14 +1327,13 @@ func testAPIBackupCreate(r *require.Assertions) { r.NotContains(out, "Connection refused") r.NotContains(out, "another operation is currently running") r.NotContains(out, "\"status\":\"error\"") - out, err = dockerExecOut("clickhouse-backup", "curl", "http://localhost:7171/metrics") + out, err = env.DockerExecOut("clickhouse-backup", "curl", "http://localhost:7171/metrics") r.NoError(err) r.Contains(out, "clickhouse_backup_last_create_status 1") - } -func fillDatabaseForAPIServer(maxTables int, minFields int, randFields int, ch *TestClickHouse, r *require.Assertions, fieldTypes []string) { - log.Info().Msgf("Create %d `long_schema`.`t%%d` tables with with %d..%d fields...", maxTables, minFields, minFields+randFields) +func fillDatabaseForAPIServer(maxTables int, minFields int, randFields int, ch *TestEnvironment, r *require.Assertions, fieldTypes []string) { + log.Debug().Msgf("Create %d `long_schema`.`t%%d` tables with with %d..%d fields...", maxTables, minFields, minFields+randFields) ch.queryWithNoError(r, "CREATE DATABASE IF NOT EXISTS long_schema") for i := 0; i < maxTables; i++ { sql := fmt.Sprintf("CREATE TABLE long_schema.t%d (id UInt64", i) @@ -968,32 +1347,31 @@ func fillDatabaseForAPIServer(maxTables int, minFields int, randFields int, ch * sql = fmt.Sprintf("INSERT INTO long_schema.t%d(id) SELECT number FROM numbers(100)", i) ch.queryWithNoError(r, sql) } - log.Info().Msg("...DONE") + log.Debug().Msg("...DONE") } func TestSkipNotExistsTable(t *testing.T) { if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "19.1") < 0 { t.Skip("TestSkipNotExistsTable too small time between `SELECT DISTINCT partition_id` and `ALTER TABLE ... FREEZE PARTITION`") } - //t.Parallel() - ch := &TestClickHouse{} - r := require.New(t) - ch.connectWithWait(r, 0*time.Second, 1*time.Second) - defer ch.chbackend.Close() + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 0*time.Second, 1*time.Second, 1*time.Minute) - log.Info().Msg("Check skip not exist errors") - ch.queryWithNoError(r, "CREATE DATABASE freeze_not_exists") + log.Debug().Msg("Check skip not exist errors") + env.queryWithNoError(r, "CREATE DATABASE freeze_not_exists") ifNotExistsCreateSQL := "CREATE TABLE IF NOT EXISTS freeze_not_exists.freeze_not_exists (id UInt64) ENGINE=MergeTree() ORDER BY id" ifNotExistsInsertSQL := "INSERT INTO freeze_not_exists.freeze_not_exists SELECT number FROM numbers(1000)" - chVersion, err := ch.chbackend.GetVersion(context.Background()) + chVersion, err := env.ch.GetVersion(context.Background()) r.NoError(err) freezeErrorHandled := false pauseChannel := make(chan int64) resumeChannel := make(chan int64) - ch.chbackend.Config.LogSQLQueries = true + if os.Getenv("TEST_LOG_LEVEL") == "debug" { + env.ch.Config.LogSQLQueries = true + } wg := sync.WaitGroup{} - wg.Add(1) + wg.Add(2) go func() { defer func() { close(pauseChannel) @@ -1003,20 +1381,20 @@ func TestSkipNotExistsTable(t *testing.T) { // pausePercent := int64(90) for i := int64(0); i < 100; i++ { testBackupName := fmt.Sprintf("not_exists_%d", i) - err = ch.chbackend.Query(ifNotExistsCreateSQL) + err = env.ch.Query(ifNotExistsCreateSQL) r.NoError(err) - err = ch.chbackend.Query(ifNotExistsInsertSQL) + err = env.ch.Query(ifNotExistsInsertSQL) r.NoError(err) if i < 5 { - log.Info().Msgf("pauseChannel <- %d", 0) + log.Debug().Msgf("pauseChannel <- %d", 0) pauseChannel <- 0 } else { - log.Info().Msgf("pauseChannel <- %d", pause/i) + log.Debug().Msgf("pauseChannel <- %d", pause/i) pauseChannel <- pause / i } startTime := time.Now() - out, err := dockerExecOut("clickhouse-backup", "bash", "-ce", "LOG_LEVEL=debug CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml clickhouse-backup create --table freeze_not_exists.freeze_not_exists "+testBackupName) - log.Info().Msg(out) + out, err := env.DockerExecOut("clickhouse-backup", "bash", "-ce", "LOG_LEVEL=debug CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml clickhouse-backup create --table freeze_not_exists.freeze_not_exists "+testBackupName) + log.Debug().Msg(out) if (err != nil && (strings.Contains(out, "can't freeze") || strings.Contains(out, "no tables for backup"))) || (err == nil && !strings.Contains(out, "can't freeze")) { parseTime := func(line string) time.Time { @@ -1042,105 +1420,336 @@ func TestSkipNotExistsTable(t *testing.T) { pause += (firstTime.Sub(startTime) + freezeTime.Sub(firstTime)).Nanoseconds() } if err != nil { - if !strings.Contains(out, "no tables for backup") { + if !strings.Contains(out, "no tables for backup") && !strings.Contains(out, "code: 473, message: Possible deadlock avoided") { assert.NoError(t, err) } } if strings.Contains(out, "code: 60") && err == nil { freezeErrorHandled = true + log.Debug().Msg("CODE 60 catched") <-resumeChannel - r.NoError(dockerExec("clickhouse-backup", "bash", "-ec", "CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml clickhouse-backup delete local "+testBackupName)) + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ec", "CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml clickhouse-backup delete local "+testBackupName) break } if err == nil { - err = dockerExec("clickhouse-backup", "bash", "-ec", "CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml clickhouse-backup delete local "+testBackupName) + err = env.DockerExec("clickhouse-backup", "bash", "-ec", "CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml clickhouse-backup delete local "+testBackupName) assert.NoError(t, err) } <-resumeChannel } }() - wg.Add(1) go func() { defer func() { close(resumeChannel) wg.Done() }() for pause := range pauseChannel { - log.Info().Msgf("%d <- pauseChannel", pause) + log.Debug().Msgf("%d <- pauseChannel", pause) if pause > 0 { pauseStart := time.Now() time.Sleep(time.Duration(pause) * time.Nanosecond) - log.Info().Msgf("pause=%s pauseStart=%s", time.Duration(pause).String(), pauseStart.String()) - err = ch.chbackend.DropTable(clickhouse.Table{Database: "freeze_not_exists", Name: "freeze_not_exists"}, ifNotExistsCreateSQL, "", false, chVersion) + log.Debug().Msgf("pause=%s pauseStart=%s", time.Duration(pause).String(), pauseStart.String()) + err = env.ch.DropTable(clickhouse.Table{Database: "freeze_not_exists", Name: "freeze_not_exists"}, ifNotExistsCreateSQL, "", false, chVersion, "") r.NoError(err) } resumeChannel <- 1 } }() wg.Wait() - r.True(freezeErrorHandled) - dropDbSQL := "DROP DATABASE freeze_not_exists" - if isAtomic, err := ch.chbackend.IsAtomic("freeze_not_exists"); err == nil && isAtomic { + r.True(freezeErrorHandled, "freezeErrorHandled false") + dropDbSQL := "DROP DATABASE IF EXISTS freeze_not_exists" + if isAtomic, err := env.ch.IsAtomic("freeze_not_exists"); err == nil && isAtomic { dropDbSQL += " SYNC" } - ch.queryWithNoError(r, dropDbSQL) + env.queryWithNoError(r, dropDbSQL) + err = env.ch.Query(dropDbSQL) + if err != nil { + log.Error().Msgf("%s error: %v", dropDbSQL, err) + } + r.NoError(err) + t.Log("TestSkipNotExistsTable DONE, ALL OK") + env.Cleanup(t, r) +} + +func TestSkipTablesAndSkipTableEngines(t *testing.T) { + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 0*time.Second, 1*time.Second, 1*time.Minute) + version, err := env.ch.GetVersion(context.Background()) + r.NoError(err) + env.queryWithNoError(r, "CREATE DATABASE test_skip_tables") + env.queryWithNoError(r, "CREATE TABLE IF NOT EXISTS test_skip_tables.test_merge_tree (id UInt64, s String) ENGINE=MergeTree() ORDER BY id") + env.queryWithNoError(r, "CREATE TABLE IF NOT EXISTS test_skip_tables.test_memory (id UInt64) ENGINE=Memory") + env.queryWithNoError(r, "CREATE MATERIALIZED VIEW IF NOT EXISTS test_skip_tables.test_mv (id UInt64) ENGINE=MergeTree() ORDER BY id AS SELECT * FROM test_skip_tables.test_merge_tree") + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") >= 0 { + query := "CREATE LIVE VIEW IF NOT EXISTS test_skip_tables.test_live_view AS SELECT count() FROM test_skip_tables.test_merge_tree" + allowExperimentalAnalyzer, err := env.ch.TurnAnalyzerOffIfNecessary(version, query, "") + r.NoError(err) + env.queryWithNoError(r, query) + r.NoError(env.ch.TurnAnalyzerOnIfNecessary(version, query, allowExperimentalAnalyzer)) + } + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 { + query := "CREATE WINDOW VIEW IF NOT EXISTS test_skip_tables.test_window_view ENGINE=MergeTree() ORDER BY s AS SELECT count(), s, tumbleStart(w_id) as w_start FROM test_skip_tables.test_merge_tree GROUP BY s, tumble(now(), INTERVAL '5' SECOND) AS w_id" + allowExperimentalAnalyzer, err := env.ch.TurnAnalyzerOffIfNecessary(version, query, "") + r.NoError(err) + env.queryWithNoError(r, query) + r.NoError(env.ch.TurnAnalyzerOnIfNecessary(version, query, allowExperimentalAnalyzer)) + } + // create + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "CLICKHOUSE_SKIP_TABLES=*.test_merge_tree clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml create skip_table_pattern") + r.Error(env.DockerExec("clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/skip_table_pattern/metadata/test_skip_tables/test_merge_tree.json")) + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/skip_table_pattern/metadata/test_skip_tables/test_memory.json") + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/skip_table_pattern/metadata/test_skip_tables/test_mv.json") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", "ls -la /var/lib/clickhouse/backup/skip_table_pattern/metadata/test_skip_tables/*inner*.json") + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") >= 0 { + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/skip_table_pattern/metadata/test_skip_tables/test_live_view.json") + } + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 { + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/skip_table_pattern/metadata/test_skip_tables/test_window_view.json") + } + + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "CLICKHOUSE_SKIP_TABLE_ENGINES=memory,materializedview,windowview,liveview clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml create skip_engines") + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/skip_engines/metadata/test_skip_tables/test_merge_tree.json") + r.Error(env.DockerExec("clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/skip_engines/metadata/test_skip_tables/test_memory.json")) + r.Error(env.DockerExec("clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/skip_engines/metadata/test_skip_tables/test_mv.json")) + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", "ls -la /var/lib/clickhouse/backup/skip_engines/metadata/test_skip_tables/*inner*.json") + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") >= 0 { + r.Error(env.DockerExec("clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/skip_engines/metadata/test_skip_tables/test_live_view.json")) + } + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 { + r.Error(env.DockerExec("clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/skip_engines/metadata/test_skip_tables/test_window_view.json")) + } + + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml delete local skip_table_pattern") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml delete local skip_engines") + + //upload + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml create test_skip_full_backup") + + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "USE_RESUMABLE_STATE=0 CLICKHOUSE_SKIP_TABLES=*.test_memory clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml upload test_skip_full_backup") + env.DockerExecNoError(r, "minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_merge_tree.json") + r.Error(env.DockerExec("minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_memory.json")) + env.DockerExecNoError(r, "minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_mv.json") + env.DockerExecNoError(r, "minio", "bash", "-ce", "ls -la /bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/*inner*.json") + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") >= 0 { + env.DockerExecNoError(r, "minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_live_view.json") + } + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 { + env.DockerExecNoError(r, "minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_window_view.json") + } + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml delete remote test_skip_full_backup") + + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "USE_RESUMABLE_STATE=0 CLICKHOUSE_SKIP_TABLE_ENGINES=memory,materializedview,liveview,windowview clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml upload test_skip_full_backup") + env.DockerExecNoError(r, "minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_merge_tree.json") + r.Error(env.DockerExec("minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_memory.json")) + r.Error(env.DockerExec("minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_mv.json")) + env.DockerExecNoError(r, "minio", "bash", "-ce", "ls -la /bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/*inner*.json") + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") >= 0 { + r.Error(env.DockerExec("minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_live_view.json")) + } + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 { + r.Error(env.DockerExec("minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_window_view.json")) + } + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml delete remote test_skip_full_backup") + + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "USE_RESUMABLE_STATE=0 clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml upload test_skip_full_backup") + env.DockerExecNoError(r, "minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_merge_tree.json") + env.DockerExecNoError(r, "minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_memory.json") + env.DockerExecNoError(r, "minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_mv.json") + env.DockerExecNoError(r, "minio", "bash", "-ce", "ls -la /bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/*inner*.json") + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") >= 0 { + env.DockerExecNoError(r, "minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_live_view.json") + } + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 { + env.DockerExecNoError(r, "minio", "ls", "-la", "/bitnami/minio/data/clickhouse/backup/cluster/0/test_skip_full_backup/metadata/test_skip_tables/test_window_view.json") + } + + //download + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml delete remote test_skip_full_backup") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "USE_RESUMABLE_STATE=0 clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml upload test_skip_full_backup") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml delete local test_skip_full_backup") + + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "USE_RESUMABLE_STATE=0 CLICKHOUSE_SKIP_TABLES=*.test_merge_tree clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml download test_skip_full_backup") + r.Error(env.DockerExec("clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_merge_tree.json")) + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_memory.json") + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_mv.json") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", "ls -la /var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/*inner*.json") + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") >= 0 { + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_live_view.json") + } + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 { + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_window_view.json") + } + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "USE_RESUMABLE_STATE=0 clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml delete local test_skip_full_backup") + + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "USE_RESUMABLE_STATE=0 CLICKHOUSE_SKIP_TABLE_ENGINES=memory,materializedview,liveview,windowview clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml download test_skip_full_backup") + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_merge_tree.json") + r.Error(env.DockerExec("clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_memory.json")) + r.Error(env.DockerExec("clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_mv.json")) + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", "ls -la /var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/*inner*.json") + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") >= 0 { + r.Error(env.DockerExec("clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_live_view.json")) + } + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 { + r.Error(env.DockerExec("clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_window_view.json")) + } + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml delete local test_skip_full_backup") + + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "USE_RESUMABLE_STATE=0 clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml download test_skip_full_backup") + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_merge_tree.json") + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_memory.json") + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_mv.json") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", "ls -la /var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/*inner*.json") + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") >= 0 { + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_live_view.json") + } + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 { + env.DockerExecNoError(r, "clickhouse-backup", "ls", "-la", "/var/lib/clickhouse/backup/test_skip_full_backup/metadata/test_skip_tables/test_window_view.json") + } + + //restore + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.1") >= 0 { + env.queryWithNoError(r, "DROP DATABASE test_skip_tables NO DELAY") + } else { + env.queryWithNoError(r, "DROP DATABASE test_skip_tables") + } + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "CLICKHOUSE_SKIP_TABLES=*.test_memory clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml restore test_skip_full_backup") + result := uint64(0) + r.NoError(env.ch.SelectSingleRowNoCtx(&result, "SELECT count() FROM system.tables WHERE database='test_skip_tables' AND name!='test_memory'")) + expectedTables := uint64(3) + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") >= 0 { + expectedTables = 4 + } + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 { + expectedTables = 6 + } + //*.inner.target.* for WINDOW VIEW created only after 22.6 + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.6") >= 0 { + expectedTables = 7 + } + r.Equal(expectedTables, result) + result = uint64(1) + r.NoError(env.ch.SelectSingleRowNoCtx(&result, "SELECT count() FROM system.tables WHERE database='test_skip_tables' AND name='test_memory'")) + r.Equal(uint64(0), result) + + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.1") >= 0 { + env.queryWithNoError(r, "DROP DATABASE test_skip_tables NO DELAY") + } else { + env.queryWithNoError(r, "DROP DATABASE test_skip_tables") + } + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "CLICKHOUSE_SKIP_TABLE_ENGINES=memory,materializedview,liveview,windowview clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml restore --schema test_skip_full_backup") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "CLICKHOUSE_SKIP_TABLE_ENGINES=memory,materializedview,liveview,windowview clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml restore --data test_skip_full_backup") + result = uint64(0) + expectedTables = uint64(2) + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 { + expectedTables = 3 + } + r.NoError(env.ch.SelectSingleRowNoCtx(&result, "SELECT count() FROM system.tables WHERE database='test_skip_tables' AND engine='MergeTree'")) + r.Equal(expectedTables, result) + result = uint64(1) + r.NoError(env.ch.SelectSingleRowNoCtx(&result, "SELECT count() FROM system.tables WHERE database='test_skip_tables' AND engine IN ('Memory','MaterializedView','LiveView','WindowView')")) + r.Equal(uint64(0), result) + + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.1") >= 0 { + env.queryWithNoError(r, "DROP DATABASE test_skip_tables NO DELAY") + } else { + env.queryWithNoError(r, "DROP DATABASE test_skip_tables") + } + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml restore test_skip_full_backup") + result = uint64(0) + r.NoError(env.ch.SelectSingleRowNoCtx(&result, "SELECT count() FROM system.tables WHERE database='test_skip_tables'")) + expectedTables = uint64(4) + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") >= 0 { + expectedTables = 5 + } + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 { + expectedTables = 7 + } + //*.inner.target.* for WINDOW VIEW created only after 22.6 + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.6") >= 0 { + expectedTables = 8 + } + r.Equal(expectedTables, result) + + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.1") >= 0 { + env.queryWithNoError(r, "DROP DATABASE test_skip_tables NO DELAY") + } else { + env.queryWithNoError(r, "DROP DATABASE test_skip_tables") + } + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml delete local test_skip_full_backup") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml delete remote test_skip_full_backup") + env.Cleanup(t, r) } func TestTablePatterns(t *testing.T) { - //t.Parallel() - ch := &TestClickHouse{} - r := require.New(t) - ch.connectWithWait(r, 500*time.Millisecond, 5*time.Second) - defer ch.chbackend.Close() + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 500*time.Millisecond, 1*time.Second, 1*time.Minute) testBackupName := "test_backup_patterns" databaseList := []string{dbNameOrdinary, dbNameAtomic} var dbNameOrdinaryTest = dbNameOrdinary + "_" + t.Name() var dbNameAtomicTest = dbNameAtomic + "_" + t.Name() - for _, createPattern := range []bool{true, false} { for _, restorePattern := range []bool{true, false} { - fullCleanup(t, r, ch, []string{testBackupName}, []string{"remote", "local"}, databaseList, false, false, "config-s3.yml") - generateTestData(t, r, ch, "S3", defaultTestData) + fullCleanup(t, r, env, []string{testBackupName}, []string{"remote", "local"}, databaseList, false, false, "config-s3.yml") + generateTestData(t, r, env, "S3", defaultTestData) if createPattern { - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create_remote", "--tables", " "+dbNameOrdinaryTest+".*", testBackupName)) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create_remote", "--tables", " "+dbNameOrdinaryTest+".*", testBackupName) + out, err := env.DockerExecOut("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "tables", "--tables", " "+dbNameOrdinaryTest+".*", testBackupName) + r.NoError(err) + r.Contains(out, dbNameOrdinaryTest) + r.NotContains(out, dbNameAtomicTest) + out, err = env.DockerExecOut("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "tables", "--remote-backup", testBackupName, "--tables", " "+dbNameOrdinaryTest+".*", testBackupName) + r.NoError(err) + r.Contains(out, dbNameOrdinaryTest) + r.NotContains(out, dbNameAtomicTest) } else { - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "create_remote", testBackupName)) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create_remote", testBackupName) + out, err := env.DockerExecOut("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "tables", testBackupName) + r.NoError(err) + r.Contains(out, dbNameOrdinaryTest) + r.Contains(out, dbNameAtomicTest) + out, err = env.DockerExecOut("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "tables", "--remote-backup", testBackupName, testBackupName) + r.NoError(err) + r.Contains(out, dbNameOrdinaryTest) + r.Contains(out, dbNameAtomicTest) } - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "delete", "local", testBackupName)) - dropDatabasesFromTestDataDataSet(t, r, ch, databaseList) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", testBackupName) + dropDatabasesFromTestDataDataSet(t, r, env, databaseList) + if restorePattern { - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "restore_remote", "--tables", " "+dbNameOrdinaryTest+".*", testBackupName)) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore_remote", "--tables", " "+dbNameOrdinaryTest+".*", testBackupName) } else { - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "restore_remote", testBackupName)) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore_remote", testBackupName) } restored := uint64(0) - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&restored, fmt.Sprintf("SELECT count() FROM system.tables WHERE database='%s'", dbNameOrdinaryTest))) + r.NoError(env.ch.SelectSingleRowNoCtx(&restored, fmt.Sprintf("SELECT count() FROM system.tables WHERE database='%s'", dbNameOrdinaryTest))) r.NotZero(restored) if createPattern || restorePattern { restored = 0 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&restored, fmt.Sprintf("SELECT count() FROM system.tables WHERE database='%s'", dbNameAtomicTest))) + r.NoError(env.ch.SelectSingleRowNoCtx(&restored, fmt.Sprintf("SELECT count() FROM system.tables WHERE database='%s'", dbNameAtomicTest))) // todo, old versions of clickhouse will return empty recordset r.Zero(restored) restored = 0 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&restored, fmt.Sprintf("SELECT count() FROM system.databases WHERE name='%s'", dbNameAtomicTest))) + r.NoError(env.ch.SelectSingleRowNoCtx(&restored, fmt.Sprintf("SELECT count() FROM system.databases WHERE name='%s'", dbNameAtomicTest))) // todo, old versions of clickhouse will return empty recordset r.Zero(restored) } else { restored = 0 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&restored, fmt.Sprintf("SELECT count() FROM system.tables WHERE database='%s'", dbNameAtomicTest))) + r.NoError(env.ch.SelectSingleRowNoCtx(&restored, fmt.Sprintf("SELECT count() FROM system.tables WHERE database='%s'", dbNameAtomicTest))) r.NotZero(restored) } - fullCleanup(t, r, ch, []string{testBackupName}, []string{"remote", "local"}, databaseList, true, true, "config-s3.yml") + fullCleanup(t, r, env, []string{testBackupName}, []string{"remote", "local"}, databaseList, true, true, "config-s3.yml") } } + env.checkObjectStorageIsEmpty(t, r, "S3") + env.Cleanup(t, r) } func TestProjections(t *testing.T) { @@ -1148,28 +1757,88 @@ func TestProjections(t *testing.T) { if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.8") == -1 { t.Skipf("Test skipped, PROJECTION available only 21.8+, current version %s", os.Getenv("CLICKHOUSE_VERSION")) } - //t.Parallel() - ch := &TestClickHouse{} - r := require.New(t) - ch.connectWithWait(r, 0*time.Second, 1*time.Second) - defer ch.chbackend.Close() + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 0*time.Second, 1*time.Second, 1*time.Minute) - r.NoError(dockerCP("config-s3.yml", "clickhouse-backup:/etc/clickhouse-backup/config.yml")) - err = ch.chbackend.Query("CREATE TABLE default.table_with_projection(dt DateTime, v UInt64, PROJECTION x (SELECT toStartOfMonth(dt) m, sum(v) GROUP BY m)) ENGINE=MergeTree() ORDER BY dt") + r.NoError(env.DockerCP("config-s3.yml", "clickhouse-backup:/etc/clickhouse-backup/config.yml")) + err = env.ch.Query("CREATE TABLE default.table_with_projection(dt DateTime, v UInt64, PROJECTION x (SELECT toStartOfMonth(dt) m, sum(v) GROUP BY m)) ENGINE=MergeTree() PARTITION BY toYYYYMMDD(dt) ORDER BY dt") r.NoError(err) - err = ch.chbackend.Query("INSERT INTO default.table_with_projection SELECT today() - INTERVAL number DAY, number FROM numbers(10)") - r.NoError(err) + env.queryWithNoError(r, "INSERT INTO default.table_with_projection SELECT today() - INTERVAL number DAY, number FROM numbers(5)") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "create_remote", "test_backup_projection_full") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "delete", "local", "test_backup_projection_full") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "restore_remote", "--rm", "test_backup_projection_full") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "delete", "local", "test_backup_projection_full") + + env.queryWithNoError(r, "INSERT INTO default.table_with_projection SELECT today() - INTERVAL number WEEK, number FROM numbers(5)") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "create_remote", "--diff-from-remote", "test_backup_projection_full", "test_backup_projection_increment") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "delete", "local", "test_backup_projection_increment") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "restore_remote", "--rm", "test_backup_projection_increment") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "create", "test_backup_projection")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "restore", "--rm", "test_backup_projection")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "delete", "local", "test_backup_projection")) var counts uint64 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&counts, "SELECT count() FROM default.table_with_projection")) + r.NoError(env.ch.SelectSingleRowNoCtx(&counts, "SELECT count() FROM default.table_with_projection")) r.Equal(uint64(10), counts) - err = ch.chbackend.Query("DROP TABLE default.table_with_projection NO DELAY") + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.9") >= 0 { + counts = 0 + r.NoError(env.ch.SelectSingleRowNoCtx(&counts, "SELECT count() FROM system.parts WHERE database='default' AND table='table_with_projection' AND has(projections,'x')")) + r.Equal(uint64(10), counts) + } + + err = env.ch.Query("DROP TABLE default.table_with_projection NO DELAY") r.NoError(err) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "delete", "remote", "test_backup_projection_increment") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "delete", "remote", "test_backup_projection_full") + + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "delete", "local", "test_backup_projection_increment") + env.Cleanup(t, r) +} + +func TestCheckSystemPartsColumns(t *testing.T) { + var err error + var version int + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "23.3") == -1 { + t.Skipf("Test skipped, system.parts_columns have inconsistency only in 23.3+, current version %s", os.Getenv("CLICKHOUSE_VERSION")) + } + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 0*time.Second, 1*time.Second, 1*time.Minute) + version, err = env.ch.GetVersion(context.Background()) + r.NoError(err) + + r.NoError(env.DockerCP("config-s3.yml", "clickhouse-backup:/etc/clickhouse-backup/config.yml")) + env.queryWithNoError(r, "CREATE DATABASE IF NOT EXISTS "+t.Name()) + + // test compatible data types + createSQL := "CREATE TABLE " + t.Name() + ".test_system_parts_columns(dt DateTime, v UInt64, e Enum('test' = 1)) ENGINE=MergeTree() ORDER BY tuple()" + env.queryWithNoError(r, createSQL) + env.queryWithNoError(r, "INSERT INTO "+t.Name()+".test_system_parts_columns SELECT today() - INTERVAL number DAY, number, 'test' FROM numbers(10)") + + env.queryWithNoError(r, "ALTER TABLE "+t.Name()+".test_system_parts_columns MODIFY COLUMN dt Nullable(DateTime('Europe/Moscow')), MODIFY COLUMN v Nullable(UInt64), MODIFY COLUMN e Enum16('test2'=1, 'test'=2)", t.Name()) + env.queryWithNoError(r, "INSERT INTO "+t.Name()+".test_system_parts_columns SELECT today() - INTERVAL number DAY, number, 'test2' FROM numbers(10)") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "create", "test_system_parts_columns") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "delete", "local", "test_system_parts_columns") + + r.NoError(env.ch.DropTable(clickhouse.Table{Database: t.Name(), Name: "test_system_parts_columns"}, createSQL, "", false, version, "")) + + // test incompatible data types + env.queryWithNoError(r, "CREATE TABLE "+t.Name()+".test_system_parts_columns(dt Date, v String) ENGINE=MergeTree() PARTITION BY dt ORDER BY tuple()") + env.queryWithNoError(r, "INSERT INTO "+t.Name()+".test_system_parts_columns SELECT today() - INTERVAL number DAY, if(number>0,'a',toString(number)) FROM numbers(2)") + + mutationSQL := "ALTER TABLE " + t.Name() + ".test_system_parts_columns MODIFY COLUMN v UInt64" + err = env.ch.QueryContext(context.Background(), mutationSQL) + if err != nil { + errStr := strings.ToLower(err.Error()) + r.True(strings.Contains(errStr, "code: 341") || strings.Contains(errStr, "code: 517") || strings.Contains(errStr, "code: 524") || strings.Contains(errStr, "timeout"), "UNKNOWN ERROR: %s", err.Error()) + log.Debug().Msgf("%s RETURN EXPECTED ERROR=%#v", mutationSQL, err) + } + env.queryWithNoError(r, "INSERT INTO "+t.Name()+".test_system_parts_columns SELECT today() - INTERVAL number DAY, number FROM numbers(10)") + r.Error(env.DockerExec("clickhouse-backup", "clickhouse-backup", "create", "test_system_parts_columns")) + r.Error(env.DockerExec("clickhouse-backup", "ls", "-lah", "/var/lib/clickhouse/backup/test_system_parts_columns")) + r.Error(env.DockerExec("clickhouse-backup", "clickhouse-backup", "delete", "local", "test_system_parts_columns")) + + r.NoError(env.ch.DropTable(clickhouse.Table{Database: t.Name(), Name: "test_system_parts_columns"}, createSQL, "", false, version, "")) + r.NoError(env.dropDatabase(t.Name())) + env.Cleanup(t, r) } func TestKeepBackupRemoteAndDiffFromRemote(t *testing.T) { @@ -1177,94 +1846,103 @@ func TestKeepBackupRemoteAndDiffFromRemote(t *testing.T) { t.Skip("Skipping Advanced integration tests...") return } - //t.Parallel() - r := require.New(t) - ch := &TestClickHouse{} - ch.connectWithWait(r, 500*time.Millisecond, 2*time.Second) + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 500*time.Millisecond, 1*time.Second, 1*time.Minute) + backupNames := make([]string, 5) for i := 0; i < 5; i++ { backupNames[i] = fmt.Sprintf("keep_remote_backup_%d", i) } databaseList := []string{dbNameOrdinary, dbNameAtomic, dbNameMySQL, dbNamePostgreSQL, Issue331Atomic, Issue331Ordinary} - fullCleanup(t, r, ch, backupNames, []string{"remote", "local"}, databaseList, false, false, "config-s3.yml") - generateTestData(t, r, ch, "S3", defaultTestData) - for i, backupName := range backupNames { - generateIncrementTestData(t, ch, r, defaultIncrementData) - if i == 0 { - r.NoError(dockerExec("clickhouse-backup", "bash", "-ce", fmt.Sprintf("BACKUPS_TO_KEEP_REMOTE=3 CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml clickhouse-backup create_remote %s", backupName))) + fullCleanup(t, r, env, backupNames, []string{"remote", "local"}, databaseList, false, false, "config-s3.yml") + incrementData := defaultIncrementData + generateTestData(t, r, env, "S3", defaultTestData) + for backupNumber, backupName := range backupNames { + if backupNumber == 0 { + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", fmt.Sprintf("BACKUPS_TO_KEEP_REMOTE=3 CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml clickhouse-backup create_remote %s", backupName)) } else { - r.NoError(dockerExec("clickhouse-backup", "bash", "-ce", fmt.Sprintf("BACKUPS_TO_KEEP_REMOTE=3 CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml clickhouse-backup create_remote --diff-from-remote=%s %s", backupNames[i-1], backupName))) + incrementData = generateIncrementTestData(t, r, env, "S3", incrementData, backupNumber) + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-ce", fmt.Sprintf("BACKUPS_TO_KEEP_REMOTE=3 CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-s3.yml clickhouse-backup create_remote --diff-from-remote=%s %s", backupNames[backupNumber-1], backupName)) } } - out, err := dockerExecOut("clickhouse-backup", "bash", "-ce", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml list local") + out, err := env.DockerExecOut("clickhouse-backup", "bash", "-ce", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml list local") r.NoError(err) - // shall not delete any backup, cause all deleted backup have links as required in other backups + // shall not delete any backup, cause all deleted backups have links as required in other backups for _, backupName := range backupNames { r.Contains(out, backupName) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", backupName)) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", backupName) } latestIncrementBackup := fmt.Sprintf("keep_remote_backup_%d", len(backupNames)-1) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "download", latestIncrementBackup)) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "--rm", latestIncrementBackup)) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "download", latestIncrementBackup) + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", "clickhouse-backup -c /etc/clickhouse-backup/config-s3.yml list local") + r.NoError(err) + prevIncrementBackup := fmt.Sprintf("keep_remote_backup_%d", len(backupNames)-2) + for _, backupName := range backupNames { + if backupName == latestIncrementBackup { + r.Contains(out, backupName) + } else if backupName == prevIncrementBackup { + r.Contains(out, "+"+backupName) + } else { + r.NotContains(out, backupName) + } + } + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "--rm", latestIncrementBackup) var res uint64 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&res, fmt.Sprintf("SELECT count() FROM `%s_%s`.`%s_%s`", Issue331Atomic, t.Name(), Issue331Atomic, t.Name()))) - r.Equal(uint64(200), res) - fullCleanup(t, r, ch, backupNames, []string{"remote", "local"}, databaseList, true, true, "config-s3.yml") + r.NoError(env.ch.SelectSingleRowNoCtx(&res, fmt.Sprintf("SELECT count() FROM `%s_%s`.`%s_%s`", Issue331Atomic, t.Name(), Issue331Atomic, t.Name()))) + r.Equal(uint64(100+20*4), res) + fullCleanup(t, r, env, []string{latestIncrementBackup}, []string{"local"}, nil, true, true, "config-s3.yml") + fullCleanup(t, r, env, backupNames, []string{"remote"}, databaseList, true, true, "config-s3.yml") + env.checkObjectStorageIsEmpty(t, r, "S3") + env.Cleanup(t, r) } func TestSyncReplicaTimeout(t *testing.T) { if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "19.11") == -1 { t.Skipf("Test skipped, SYNC REPLICA ignore receive_timeout for %s version", os.Getenv("CLICKHOUSE_VERSION")) } - //t.Parallel() - r := require.New(t) - ch := &TestClickHouse{} - ch.connectWithWait(r, 0*time.Millisecond, 2*time.Second) - defer ch.chbackend.Close() + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 0*time.Millisecond, 1*time.Second, 1*time.Minute) - createDbSQL := "CREATE DATABASE IF NOT EXISTS " + t.Name() - ch.queryWithNoError(r, createDbSQL) + env.queryWithNoError(r, "CREATE DATABASE IF NOT EXISTS "+t.Name()) dropReplTables := func() { for _, table := range []string{"repl1", "repl2"} { query := "DROP TABLE IF EXISTS " + t.Name() + "." + table if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.3") == 1 { query += " NO DELAY" } - ch.queryWithNoError(r, query) + env.queryWithNoError(r, query) } } dropReplTables() - ch.queryWithNoError(r, "CREATE TABLE "+t.Name()+".repl1 (v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/"+t.Name()+"/repl','repl1') ORDER BY tuple()") - ch.queryWithNoError(r, "CREATE TABLE "+t.Name()+".repl2 (v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/"+t.Name()+"/repl','repl2') ORDER BY tuple()") + env.queryWithNoError(r, "CREATE TABLE "+t.Name()+".repl1 (v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/"+t.Name()+"/repl','repl1') ORDER BY tuple()") + env.queryWithNoError(r, "CREATE TABLE "+t.Name()+".repl2 (v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/"+t.Name()+"/repl','repl2') ORDER BY tuple()") - ch.queryWithNoError(r, "INSERT INTO "+t.Name()+".repl1 SELECT number FROM numbers(10)") + env.queryWithNoError(r, "INSERT INTO "+t.Name()+".repl1 SELECT number FROM numbers(10)") - ch.queryWithNoError(r, "SYSTEM STOP REPLICATED SENDS "+t.Name()+".repl1") - ch.queryWithNoError(r, "SYSTEM STOP FETCHES "+t.Name()+".repl2") + env.queryWithNoError(r, "SYSTEM STOP REPLICATED SENDS "+t.Name()+".repl1") + env.queryWithNoError(r, "SYSTEM STOP FETCHES "+t.Name()+".repl2") - ch.queryWithNoError(r, "INSERT INTO "+t.Name()+".repl1 SELECT number FROM numbers(100)") + env.queryWithNoError(r, "INSERT INTO "+t.Name()+".repl1 SELECT number FROM numbers(100)") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "--tables="+t.Name()+".repl*", "test_not_synced_backup")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "upload", "test_not_synced_backup")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_not_synced_backup")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "remote", "test_not_synced_backup")) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "--tables="+t.Name()+".repl*", "test_not_synced_backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "upload", "test_not_synced_backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_not_synced_backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "remote", "test_not_synced_backup") - ch.queryWithNoError(r, "SYSTEM START REPLICATED SENDS "+t.Name()+".repl1") - ch.queryWithNoError(r, "SYSTEM START FETCHES "+t.Name()+".repl2") + env.queryWithNoError(r, "SYSTEM START REPLICATED SENDS "+t.Name()+".repl1") + env.queryWithNoError(r, "SYSTEM START FETCHES "+t.Name()+".repl2") dropReplTables() - r.NoError(ch.dropDatabase(t.Name())) + r.NoError(env.dropDatabase(t.Name())) + env.Cleanup(t, r) } func TestGetPartitionId(t *testing.T) { if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "19.17") == -1 { t.Skipf("Test skipped, is_in_partition_key not available for %s version", os.Getenv("CLICKHOUSE_VERSION")) } - //t.Parallel() - r := require.New(t) - ch := &TestClickHouse{} - ch.connectWithWait(r, 500*time.Millisecond, 1*time.Second) - defer ch.chbackend.Close() + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 500*time.Millisecond, 1*time.Second, 1*time.Minute) type testData struct { CreateTableSQL string @@ -1316,25 +1994,21 @@ func TestGetPartitionId(t *testing.T) { "", }, } - if isAtomic, _ := ch.chbackend.IsAtomic("default"); !isAtomic { + if isAtomic, _ := env.ch.IsAtomic("default"); !isAtomic { testCases[0].CreateTableSQL = strings.Replace(testCases[0].CreateTableSQL, "UUID 'b45e751f-6c06-42a3-ab4a-f5bb9ac3716e'", "", 1) } for _, tc := range testCases { - partitionId, partitionName, err := partition.GetPartitionIdAndName(context.Background(), ch.chbackend, tc.Database, tc.Table, tc.CreateTableSQL, tc.Partition) + partitionId, partitionName, err := partition.GetPartitionIdAndName(context.Background(), env.ch, tc.Database, tc.Table, tc.CreateTableSQL, tc.Partition) assert.NoError(t, err) assert.Equal(t, tc.ExpectedId, partitionId) assert.Equal(t, tc.ExpectedName, partitionName) } + env.Cleanup(t, r) } func TestRestoreMutationInProgress(t *testing.T) { - //t.Parallel() - r := require.New(t) - ch := &TestClickHouse{} - ch.connectWithWait(r, 0*time.Second, 5*time.Second) - defer ch.chbackend.Close() - version, err := ch.chbackend.GetVersion(context.Background()) - r.NoError(err) + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 0*time.Second, 1*time.Second, 1*time.Minute) zkPath := "/clickhouse/tables/{shard}/" + t.Name() + "/test_restore_mutation_in_progress" onCluster := "" if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.8") >= 0 { @@ -1345,24 +2019,26 @@ func TestRestoreMutationInProgress(t *testing.T) { onCluster = " ON CLUSTER '{cluster}'" } createDbSQL := "CREATE DATABASE IF NOT EXISTS " + t.Name() - ch.queryWithNoError(r, createDbSQL) + env.queryWithNoError(r, createDbSQL) + version, err := env.ch.GetVersion(context.Background()) + r.NoError(err) createSQL := fmt.Sprintf("CREATE TABLE %s.test_restore_mutation_in_progress %s (id UInt64, attr String) ENGINE=ReplicatedMergeTree('%s','{replica}') PARTITION BY id ORDER BY id", t.Name(), onCluster, zkPath) - ch.queryWithNoError(r, createSQL) - ch.queryWithNoError(r, "INSERT INTO "+t.Name()+".test_restore_mutation_in_progress SELECT number, if(number>0,'a',toString(number)) FROM numbers(2)") + env.queryWithNoError(r, createSQL) + env.queryWithNoError(r, "INSERT INTO "+t.Name()+".test_restore_mutation_in_progress SELECT number, if(number>0,'a',toString(number)) FROM numbers(2)") mutationSQL := "ALTER TABLE " + t.Name() + ".test_restore_mutation_in_progress MODIFY COLUMN attr UInt64" - err = ch.chbackend.QueryContext(context.Background(), mutationSQL) + err = env.ch.QueryContext(context.Background(), mutationSQL) if err != nil { errStr := strings.ToLower(err.Error()) r.True(strings.Contains(errStr, "code: 341") || strings.Contains(errStr, "code: 517") || strings.Contains(errStr, "timeout"), "UNKNOWN ERROR: %s", err.Error()) - t.Logf("%s RETURN EXPECTED ERROR=%#v", mutationSQL, err) + log.Debug().Msgf("%s RETURN EXPECTED ERROR=%#v", mutationSQL, err) } attrs := make([]struct { Attr uint64 `ch:"attr"` }, 0) - err = ch.chbackend.Select(&attrs, "SELECT attr FROM "+t.Name()+".test_restore_mutation_in_progress ORDER BY id") + err = env.ch.Select(&attrs, "SELECT attr FROM "+t.Name()+".test_restore_mutation_in_progress ORDER BY id") r.NotEqual(nil, err) errStr := strings.ToLower(err.Error()) r.True(strings.Contains(errStr, "code: 53") || strings.Contains(errStr, "code: 6")) @@ -1370,29 +2046,29 @@ func TestRestoreMutationInProgress(t *testing.T) { if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.8") >= 0 { mutationSQL = "ALTER TABLE " + t.Name() + ".test_restore_mutation_in_progress RENAME COLUMN attr TO attr_1" - err = ch.chbackend.QueryContext(context.Background(), mutationSQL) + err = env.ch.QueryContext(context.Background(), mutationSQL) r.NotEqual(nil, err) errStr = strings.ToLower(err.Error()) r.True(strings.Contains(errStr, "code: 517") || strings.Contains(errStr, "timeout")) - t.Logf("%s RETURN EXPECTED ERROR=%#v", mutationSQL, err) + log.Debug().Msgf("%s RETURN EXPECTED ERROR=%#v", mutationSQL, err) } - r.NoError(dockerExec("clickhouse", "clickhouse", "client", "-q", "SELECT * FROM system.mutations WHERE is_done=0 FORMAT Vertical")) + env.DockerExecNoError(r, "clickhouse", "clickhouse", "client", "-q", "SELECT * FROM system.mutations WHERE is_done=0 FORMAT Vertical") // backup with check consistency - out, createErr := dockerExecOut("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "--tables="+t.Name()+".test_restore_mutation_in_progress", "test_restore_mutation_in_progress") + out, createErr := env.DockerExecOut("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "--tables="+t.Name()+".test_restore_mutation_in_progress", "test_restore_mutation_in_progress") r.NotEqual(createErr, nil) r.Contains(out, "have inconsistent data types") - t.Log(out) + log.Debug().Msg(out) // backup without check consistency - out, createErr = dockerExecOut("clickhouse-backup", "clickhouse-backup", "create", "-c", "/etc/clickhouse-backup/config-s3.yml", "--skip-check-parts-columns", "--tables="+t.Name()+".test_restore_mutation_in_progress", "test_restore_mutation_in_progress") - t.Log(out) + out, createErr = env.DockerExecOut("clickhouse-backup", "clickhouse-backup", "create", "-c", "/etc/clickhouse-backup/config-s3.yml", "--skip-check-parts-columns", "--tables="+t.Name()+".test_restore_mutation_in_progress", "test_restore_mutation_in_progress") + log.Debug().Msg(out) r.NoError(createErr) r.NotContains(out, "have inconsistent data types") - r.NoError(ch.chbackend.DropTable(clickhouse.Table{Database: t.Name(), Name: "test_restore_mutation_in_progress"}, "", "", false, version)) + r.NoError(env.ch.DropTable(clickhouse.Table{Database: t.Name(), Name: "test_restore_mutation_in_progress"}, "", "", false, version, "")) var restoreErr error - restoreErr = dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "--rm", "--tables="+t.Name()+".test_restore_mutation_in_progress", "test_restore_mutation_in_progress") + restoreErr = env.DockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "--rm", "--tables="+t.Name()+".test_restore_mutation_in_progress", "test_restore_mutation_in_progress") if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.8") >= 0 && compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.8") < 0 { r.NotEqual(restoreErr, nil) } else { @@ -1409,7 +2085,7 @@ func TestRestoreMutationInProgress(t *testing.T) { } } selectSQL := fmt.Sprintf("SELECT %s FROM "+t.Name()+".test_restore_mutation_in_progress ORDER BY id", checkRestoredData) - selectErr := ch.chbackend.Select(&attrs, selectSQL) + selectErr := env.ch.Select(&attrs, selectSQL) expectedSelectResults := make([]struct { Attr uint64 `ch:"attr"` }, 1) @@ -1436,122 +2112,133 @@ func TestRestoreMutationInProgress(t *testing.T) { if expectedSelectError != "" { r.Error(selectErr) r.Contains(strings.ToLower(selectErr.Error()), expectedSelectError) - t.Logf("%s RETURN EXPECTED ERROR=%#v", selectSQL, selectErr) + log.Debug().Msgf("%s RETURN EXPECTED ERROR=%#v", selectSQL, selectErr) } else { r.NoError(selectErr) } - r.NoError(dockerExec("clickhouse", "clickhouse", "client", "-q", "SELECT * FROM system.mutations FORMAT Vertical")) + env.DockerExecNoError(r, "clickhouse", "clickhouse", "client", "-q", "SELECT * FROM system.mutations FORMAT Vertical") - r.NoError(ch.chbackend.DropTable(clickhouse.Table{Database: t.Name(), Name: "test_restore_mutation_in_progress"}, "", "", false, version)) - r.NoError(ch.dropDatabase(t.Name())) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_restore_mutation_in_progress")) + r.NoError(env.ch.DropTable(clickhouse.Table{Database: t.Name(), Name: "test_restore_mutation_in_progress"}, "", "", false, version, "")) + r.NoError(env.dropDatabase(t.Name())) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_restore_mutation_in_progress") + env.Cleanup(t, r) } func TestInnerTablesMaterializedView(t *testing.T) { - //t.Parallel() - ch := &TestClickHouse{} - r := require.New(t) - ch.connectWithWait(r, 1*time.Second, 10*time.Second) - defer ch.chbackend.Close() - - ch.queryWithNoError(r, "CREATE DATABASE test_mv") - ch.queryWithNoError(r, "CREATE TABLE test_mv.src_table (v UInt64) ENGINE=MergeTree() ORDER BY v") - ch.queryWithNoError(r, "CREATE TABLE test_mv.dst_table (v UInt64) ENGINE=MergeTree() ORDER BY v") - ch.queryWithNoError(r, "CREATE MATERIALIZED VIEW test_mv.mv_with_inner (v UInt64) ENGINE=MergeTree() ORDER BY v AS SELECT v FROM test_mv.src_table") - ch.queryWithNoError(r, "CREATE MATERIALIZED VIEW test_mv.mv_with_dst TO test_mv.dst_table AS SELECT v FROM test_mv.src_table") - ch.queryWithNoError(r, "INSERT INTO test_mv.src_table SELECT number FROM numbers(100)") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "create", "test_mv", "--tables=test_mv.mv_with*,test_mv.dst*")) + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 1*time.Second, 1*time.Second, 1*time.Minute) + + env.queryWithNoError(r, "CREATE DATABASE test_mv") + env.queryWithNoError(r, "CREATE TABLE test_mv.src_table (v UInt64) ENGINE=MergeTree() ORDER BY v") + env.queryWithNoError(r, "CREATE TABLE test_mv.dst_table (v UInt64) ENGINE=MergeTree() ORDER BY v") + env.queryWithNoError(r, "CREATE MATERIALIZED VIEW test_mv.mv_with_inner (v UInt64) ENGINE=MergeTree() ORDER BY v AS SELECT v FROM test_mv.src_table") + env.queryWithNoError(r, "CREATE MATERIALIZED VIEW test_mv.mv_with_dst TO test_mv.dst_table AS SELECT v FROM test_mv.src_table") + env.queryWithNoError(r, "INSERT INTO test_mv.src_table SELECT number FROM numbers(100)") + + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "test_mv", "--tables=test_mv.mv_with*,test_mv.dst*") dropSQL := "DROP DATABASE test_mv" - isAtomic, err := ch.chbackend.IsAtomic("test_mv") + isAtomic, err := env.ch.IsAtomic("test_mv") r.NoError(err) if isAtomic { dropSQL += " NO DELAY" } - ch.queryWithNoError(r, dropSQL) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "test_mv", "--tables=test_mv.mv_with*,test_mv.dst*")) + env.queryWithNoError(r, dropSQL) var rowCnt uint64 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&rowCnt, "SELECT count() FROM test_mv.mv_with_inner")) + + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "test_mv", "--tables=test_mv.mv_with*,test_mv.dst*") + r.NoError(env.ch.SelectSingleRowNoCtx(&rowCnt, "SELECT count() FROM test_mv.mv_with_inner")) + r.Equal(uint64(100), rowCnt) + r.NoError(env.ch.SelectSingleRowNoCtx(&rowCnt, "SELECT count() FROM test_mv.mv_with_dst")) r.Equal(uint64(100), rowCnt) - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&rowCnt, "SELECT count() FROM test_mv.mv_with_dst")) + + r.NoError(env.dropDatabase("test_mv")) + // https://github.com/Altinity/clickhouse-backup/issues/777 + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "upload", "test_mv", "--delete-source", "--tables=test_mv.mv_with*,test_mv.dst*") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "download", "test_mv", "--tables=test_mv.mv_with*,test_mv.dst*") + + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "test_mv", "--tables=test_mv.mv_with*,test_mv.dst*") + r.NoError(env.ch.SelectSingleRowNoCtx(&rowCnt, "SELECT count() FROM test_mv.mv_with_inner")) r.Equal(uint64(100), rowCnt) - r.NoError(ch.dropDatabase("test_mv")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_mv")) + r.NoError(env.ch.SelectSingleRowNoCtx(&rowCnt, "SELECT count() FROM test_mv.mv_with_dst")) + r.Equal(uint64(100), rowCnt) + + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_mv") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "remote", "test_mv") + env.Cleanup(t, r) } func TestFIPS(t *testing.T) { if os.Getenv("QA_AWS_ACCESS_KEY") == "" { t.Skip("QA_AWS_ACCESS_KEY is empty, TestFIPS will skip") } - //t.Parallel() - ch := &TestClickHouse{} - r := require.New(t) - ch.connectWithWait(r, 1*time.Second, 10*time.Second) - defer ch.chbackend.Close() + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 1*time.Second, 1*time.Second, 1*time.Minute) fipsBackupName := fmt.Sprintf("fips_backup_%d", rand.Int()) - r.NoError(dockerExec("clickhouse", "rm", "-fv", "/etc/apt/sources.list.d/clickhouse.list")) - installDebIfNotExists(r, "clickhouse", "ca-certificates", "curl", "gettext-base", "bsdmainutils", "dnsutils", "git") - r.NoError(dockerExec("clickhouse", "update-ca-certificates")) - r.NoError(dockerCP("config-s3-fips.yml", "clickhouse:/etc/clickhouse-backup/config.yml.fips-template")) - r.NoError(dockerExec("clickhouse", "git", "clone", "--depth", "1", "https://github.com/drwetter/testssl.sh.git", "/opt/testssl")) - r.NoError(dockerExec("clickhouse", "chmod", "+x", "/opt/testssl/testssl.sh")) + env.DockerExecNoError(r, "clickhouse", "rm", "-fv", "/etc/apt/sources.list.d/clickhouse.list") + env.InstallDebIfNotExists(r, "clickhouse", "ca-certificates", "curl", "gettext-base", "bsdmainutils", "dnsutils", "git") + env.DockerExecNoError(r, "clickhouse", "update-ca-certificates") + r.NoError(env.DockerCP("config-s3-fips.yml", "clickhouse:/etc/clickhouse-backup/config.yml.fips-template")) + env.DockerExecNoError(r, "clickhouse", "git", "clone", "--depth", "1", "--branch", "v3.2rc3", "https://github.com/drwetter/testssl.sh.git", "/opt/testssl") + env.DockerExecNoError(r, "clickhouse", "chmod", "+x", "/opt/testssl/testssl.sh") generateCerts := func(certType, keyLength, curveType string) { - r.NoError(dockerExec("clickhouse", "bash", "-xce", "openssl rand -out /root/.rnd 2048")) + env.DockerExecNoError(r, "clickhouse", "bash", "-xce", "openssl rand -out /root/.rnd 2048") switch certType { case "rsa": - r.NoError(dockerExec("clickhouse", "bash", "-xce", fmt.Sprintf("openssl genrsa -out /etc/clickhouse-backup/ca-key.pem %s", keyLength))) - r.NoError(dockerExec("clickhouse", "bash", "-xce", fmt.Sprintf("openssl genrsa -out /etc/clickhouse-backup/server-key.pem %s", keyLength))) + env.DockerExecNoError(r, "clickhouse", "bash", "-xce", fmt.Sprintf("openssl genrsa -out /etc/clickhouse-backup/ca-key.pem %s", keyLength)) + env.DockerExecNoError(r, "clickhouse", "bash", "-xce", fmt.Sprintf("openssl genrsa -out /etc/clickhouse-backup/server-key.pem %s", keyLength)) case "ecdsa": - r.NoError(dockerExec("clickhouse", "bash", "-xce", fmt.Sprintf("openssl ecparam -name %s -genkey -out /etc/clickhouse-backup/ca-key.pem", curveType))) - r.NoError(dockerExec("clickhouse", "bash", "-xce", fmt.Sprintf("openssl ecparam -name %s -genkey -out /etc/clickhouse-backup/server-key.pem", curveType))) + env.DockerExecNoError(r, "clickhouse", "bash", "-xce", fmt.Sprintf("openssl ecparam -name %s -genkey -out /etc/clickhouse-backup/ca-key.pem", curveType)) + env.DockerExecNoError(r, "clickhouse", "bash", "-xce", fmt.Sprintf("openssl ecparam -name %s -genkey -out /etc/clickhouse-backup/server-key.pem", curveType)) } - r.NoError(dockerExec("clickhouse", "bash", "-xce", "openssl req -subj \"/O=altinity\" -x509 -new -nodes -key /etc/clickhouse-backup/ca-key.pem -sha256 -days 365000 -out /etc/clickhouse-backup/ca-cert.pem")) - r.NoError(dockerExec("clickhouse", "bash", "-xce", "openssl req -subj \"/CN=localhost\" -addext \"subjectAltName = DNS:localhost,DNS:*.cluster.local\" -new -key /etc/clickhouse-backup/server-key.pem -out /etc/clickhouse-backup/server-req.csr")) - r.NoError(dockerExec("clickhouse", "bash", "-xce", "openssl x509 -req -days 365000 -extensions SAN -extfile <(printf \"\\n[SAN]\\nsubjectAltName=DNS:localhost,DNS:*.cluster.local\") -in /etc/clickhouse-backup/server-req.csr -out /etc/clickhouse-backup/server-cert.pem -CA /etc/clickhouse-backup/ca-cert.pem -CAkey /etc/clickhouse-backup/ca-key.pem -CAcreateserial")) + env.DockerExecNoError(r, "clickhouse", "bash", "-xce", "openssl req -subj \"/O=altinity\" -x509 -new -nodes -key /etc/clickhouse-backup/ca-key.pem -sha256 -days 365000 -out /etc/clickhouse-backup/ca-cert.pem") + env.DockerExecNoError(r, "clickhouse", "bash", "-xce", "openssl req -subj \"/CN=localhost\" -addext \"subjectAltName = DNS:localhost,DNS:*.cluster.local\" -new -key /etc/clickhouse-backup/server-key.pem -out /etc/clickhouse-backup/server-req.csr") + env.DockerExecNoError(r, "clickhouse", "bash", "-xce", "openssl x509 -req -days 365000 -extensions SAN -extfile <(printf \"\\n[SAN]\\nsubjectAltName=DNS:localhost,DNS:*.cluster.local\") -in /etc/clickhouse-backup/server-req.csr -out /etc/clickhouse-backup/server-cert.pem -CA /etc/clickhouse-backup/ca-cert.pem -CAkey /etc/clickhouse-backup/ca-key.pem -CAcreateserial") } - r.NoError(dockerExec("clickhouse", "bash", "-xec", "cat /etc/clickhouse-backup/config-s3-fips.yml.template | envsubst > /etc/clickhouse-backup/config-s3-fips.yml")) + env.DockerExecNoError(r, "clickhouse", "bash", "-xec", "cat /etc/clickhouse-backup/config-s3-fips.yml.template | envsubst > /etc/clickhouse-backup/config-s3-fips.yml") generateCerts("rsa", "4096", "") - ch.queryWithNoError(r, "CREATE DATABASE "+t.Name()) + env.queryWithNoError(r, "CREATE DATABASE "+t.Name()) createSQL := "CREATE TABLE " + t.Name() + ".fips_table (v UInt64) ENGINE=MergeTree() ORDER BY tuple()" - ch.queryWithNoError(r, createSQL) - ch.queryWithNoError(r, "INSERT INTO "+t.Name()+".fips_table SELECT number FROM numbers(1000)") - r.NoError(dockerExec("clickhouse", "bash", "-ce", "clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml create_remote --tables="+t.Name()+".fips_table "+fipsBackupName)) - r.NoError(dockerExec("clickhouse", "bash", "-ce", "clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml delete local "+fipsBackupName)) - r.NoError(dockerExec("clickhouse", "bash", "-ce", "clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml restore_remote --tables="+t.Name()+".fips_table "+fipsBackupName)) - r.NoError(dockerExec("clickhouse", "bash", "-ce", "clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml delete local "+fipsBackupName)) - r.NoError(dockerExec("clickhouse", "bash", "-ce", "clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml delete remote "+fipsBackupName)) - - log.Info().Msg("Run `clickhouse-backup-fips server` in background") - r.NoError(dockerExec("-d", "clickhouse", "bash", "-ce", "AWS_USE_FIPS_ENDPOINT=true clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml server &>>/tmp/clickhouse-backup-server-fips.log")) + env.queryWithNoError(r, createSQL) + env.queryWithNoError(r, "INSERT INTO "+t.Name()+".fips_table SELECT number FROM numbers(1000)") + env.DockerExecNoError(r, "clickhouse", "bash", "-ce", "clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml create_remote --tables="+t.Name()+".fips_table "+fipsBackupName) + env.DockerExecNoError(r, "clickhouse", "bash", "-ce", "clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml delete local "+fipsBackupName) + env.DockerExecNoError(r, "clickhouse", "bash", "-ce", "clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml restore_remote --tables="+t.Name()+".fips_table "+fipsBackupName) + env.DockerExecNoError(r, "clickhouse", "bash", "-ce", "clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml delete local "+fipsBackupName) + env.DockerExecNoError(r, "clickhouse", "bash", "-ce", "clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml delete remote "+fipsBackupName) + + log.Debug().Msg("Run `clickhouse-backup-fips server` in background") + env.DockerExecBackgroundNoError(r, "clickhouse", "bash", "-ce", "AWS_USE_FIPS_ENDPOINT=true clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml server &>>/tmp/clickhouse-backup-server-fips.log") time.Sleep(1 * time.Second) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{fmt.Sprintf("create_remote --tables="+t.Name()+".fips_table %s", fipsBackupName)}, true) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{fmt.Sprintf("delete local %s", fipsBackupName)}, false) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{fmt.Sprintf("restore_remote --tables="+t.Name()+".fips_table %s", fipsBackupName)}, true) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{fmt.Sprintf("delete local %s", fipsBackupName)}, false) - runClickHouseClientInsertSystemBackupActions(r, ch, []string{fmt.Sprintf("delete remote %s", fipsBackupName)}, false) + runClickHouseClientInsertSystemBackupActions(r, env, []string{fmt.Sprintf("create_remote --tables="+t.Name()+".fips_table %s", fipsBackupName)}, true) + runClickHouseClientInsertSystemBackupActions(r, env, []string{fmt.Sprintf("delete local %s", fipsBackupName)}, false) + runClickHouseClientInsertSystemBackupActions(r, env, []string{fmt.Sprintf("restore_remote --tables="+t.Name()+".fips_table %s", fipsBackupName)}, true) + runClickHouseClientInsertSystemBackupActions(r, env, []string{fmt.Sprintf("delete local %s", fipsBackupName)}, false) + runClickHouseClientInsertSystemBackupActions(r, env, []string{fmt.Sprintf("delete remote %s", fipsBackupName)}, false) inProgressActions := make([]struct { Command string `ch:"command"` Status string `ch:"status"` }, 0) - r.NoError(ch.chbackend.StructSelect(&inProgressActions, + r.NoError(env.ch.StructSelect(&inProgressActions, "SELECT command, status FROM system.backup_actions WHERE command LIKE ? AND status IN (?,?)", fmt.Sprintf("%%%s%%", fipsBackupName), status.InProgressStatus, status.ErrorStatus, )) r.Equal(0, len(inProgressActions), "inProgressActions=%+v", inProgressActions) - r.NoError(dockerExec("clickhouse", "pkill", "-n", "-f", "clickhouse-backup-fips")) + env.DockerExecNoError(r, "clickhouse", "pkill", "-n", "-f", "clickhouse-backup-fips") testTLSCerts := func(certType, keyLength, curveName string, cipherList ...string) { generateCerts(certType, keyLength, curveName) - log.Info().Msgf("Run `clickhouse-backup-fips server` in background for %s %s %s", certType, keyLength, curveName) - r.NoError(dockerExec("-d", "clickhouse", "bash", "-ce", "AWS_USE_FIPS_ENDPOINT=true clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml server &>>/tmp/clickhouse-backup-server-fips.log")) + log.Debug().Msgf("Run `clickhouse-backup-fips server` in background for %s %s %s", certType, keyLength, curveName) + env.DockerExecBackgroundNoError(r, "clickhouse", "bash", "-ce", "AWS_USE_FIPS_ENDPOINT=true clickhouse-backup-fips -c /etc/clickhouse-backup/config-s3-fips.yml server &>>/tmp/clickhouse-backup-server-fips.log") time.Sleep(1 * time.Second) - r.NoError(dockerExec("clickhouse", "bash", "-ce", "rm -rf /tmp/testssl* && /opt/testssl/testssl.sh -e -s -oC /tmp/testssl.csv --color 0 --disable-rating --quiet -n min --mode parallel --add-ca /etc/clickhouse-backup/ca-cert.pem localhost:7172")) - out, err := dockerExecOut("clickhouse", "bash", "-ce", fmt.Sprintf("grep -c -E '%s' /tmp/testssl.csv", strings.Join(cipherList, "|"))) + env.DockerExecNoError(r, "clickhouse", "bash", "-ce", "rm -rf /tmp/testssl* && /opt/testssl/testssl.sh -e -s -oC /tmp/testssl.csv --color 0 --disable-rating --quiet -n min --mode parallel --add-ca /etc/clickhouse-backup/ca-cert.pem localhost:7172") + env.DockerExecNoError(r, "clickhouse", "cat", "/tmp/testssl.csv") + out, err := env.DockerExecOut("clickhouse", "bash", "-ce", fmt.Sprintf("grep -o -E '%s' /tmp/testssl.csv | uniq | wc -l", strings.Join(cipherList, "|"))) r.NoError(err) r.Equal(strconv.Itoa(len(cipherList)), strings.Trim(out, " \t\r\n")) @@ -1559,377 +2246,372 @@ func TestFIPS(t *testing.T) { Command string `ch:"command"` Status string `ch:"status"` }, 0) - r.NoError(ch.chbackend.StructSelect(&inProgressActions, + r.NoError(env.ch.StructSelect(&inProgressActions, "SELECT command, status FROM system.backup_actions WHERE command LIKE ? AND status IN (?,?)", fmt.Sprintf("%%%s%%", fipsBackupName), status.InProgressStatus, status.ErrorStatus, )) r.Equal(0, len(inProgressActions), "inProgressActions=%+v", inProgressActions) - r.NoError(dockerExec("clickhouse", "pkill", "-n", "-f", "clickhouse-backup-fips")) + env.DockerExecNoError(r, "clickhouse", "pkill", "-n", "-f", "clickhouse-backup-fips") } // https://www.perplexity.ai/search/0920f1e8-59ec-4e14-b779-ba7b2e037196 - testTLSCerts("rsa", "4096", "", "ECDHE-RSA-AES128-GCM-SHA256", "ECDHE-RSA-AES256-GCM-SHA384", "AES128-GCM-SHA256", "AES256-GCM-SHA384") + testTLSCerts("rsa", "4096", "", "ECDHE-RSA-AES128-GCM-SHA256", "ECDHE-RSA-AES256-GCM-SHA384", "AES_128_GCM_SHA256", "AES_256_GCM_SHA384") testTLSCerts("ecdsa", "", "prime256v1", "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-ECDSA-AES256-GCM-SHA384") - r.NoError(ch.chbackend.DropTable(clickhouse.Table{Database: t.Name(), Name: "fips_table"}, createSQL, "", false, 0)) - r.NoError(ch.dropDatabase(t.Name())) - -} - -func TestIntegrationS3Glacier(t *testing.T) { - if isTestShouldSkip("GLACIER_TESTS") { - t.Skip("Skipping GLACIER integration tests...") - return - } - r := require.New(t) - r.NoError(dockerCP("config-s3-glacier.yml", "clickhouse-backup:/etc/clickhouse-backup/config.yml.s3glacier-template")) - installDebIfNotExists(r, "clickhouse-backup", "curl", "gettext-base", "bsdmainutils", "dnsutils", "git", "ca-certificates") - r.NoError(dockerExec("clickhouse-backup", "bash", "-xec", "cat /etc/clickhouse-backup/config.yml.s3glacier-template | envsubst > /etc/clickhouse-backup/config-s3-glacier.yml")) - dockerExecTimeout = 60 * time.Minute - runMainIntegrationScenario(t, "GLACIER", "config-s3-glacier.yml") - dockerExecTimeout = 3 * time.Minute -} - -func TestIntegrationS3(t *testing.T) { - //t.Parallel() - runMainIntegrationScenario(t, "S3", "config-s3.yml") -} - -func TestIntegrationGCS(t *testing.T) { - if isTestShouldSkip("GCS_TESTS") { - t.Skip("Skipping GCS integration tests...") - return - } - //t.Parallel() - runMainIntegrationScenario(t, "GCS", "config-gcs.yml") -} - -func TestIntegrationAzure(t *testing.T) { - if isTestShouldSkip("AZURE_TESTS") { - t.Skip("Skipping Azure integration tests...") - return - } - //t.Parallel() - runMainIntegrationScenario(t, "AZBLOB", "config-azblob.yml") -} - -func TestIntegrationSFTPAuthPassword(t *testing.T) { - //t.Parallel() - runMainIntegrationScenario(t, "SFTP", "config-sftp-auth-password.yaml") -} - -func TestIntegrationFTP(t *testing.T) { - //t.Parallel() - runMainIntegrationScenario(t, "FTP", "config-ftp.yaml") -} - -func TestIntegrationSFTPAuthKey(t *testing.T) { - uploadSSHKeys(require.New(t), "clickhouse-backup") - //t.Parallel() - runMainIntegrationScenario(t, "SFTP", "config-sftp-auth-key.yaml") -} - -func TestIntegrationCustomKopia(t *testing.T) { - //t.Parallel() - r := require.New(t) - runIntegrationCustom(t, r, "kopia") -} -func TestIntegrationCustomRestic(t *testing.T) { - //t.Parallel() - r := require.New(t) - runIntegrationCustom(t, r, "restic") -} - -func TestIntegrationCustomRsync(t *testing.T) { - r := require.New(t) - uploadSSHKeys(r, "clickhouse-backup") - //t.Parallel() - runIntegrationCustom(t, r, "rsync") + r.NoError(env.ch.DropTable(clickhouse.Table{Database: t.Name(), Name: "fips_table"}, createSQL, "", false, 0, "")) + r.NoError(env.dropDatabase(t.Name())) + env.Cleanup(t, r) } -func runIntegrationCustom(t *testing.T, r *require.Assertions, customType string) { - r.NoError(dockerExec("clickhouse-backup", "mkdir", "-pv", "/custom/"+customType)) - r.NoError(dockerCP("./"+customType+"/", "clickhouse-backup:/custom/")) - runMainIntegrationScenario(t, "CUSTOM", "config-custom-"+customType+".yml") -} -func TestIntegrationEmbedded(t *testing.T) { - //t.Skipf("Test skipped, wait 23.8, RESTORE Ordinary table and RESTORE MATERIALIZED VIEW and {uuid} not works for %s version, look https://github.com/ClickHouse/ClickHouse/issues/43971 and https://github.com/ClickHouse/ClickHouse/issues/42709", os.Getenv("CLICKHOUSE_VERSION")) - //dependencies restore https://github.com/ClickHouse/ClickHouse/issues/39416, fixed in 23.3 - version := os.Getenv("CLICKHOUSE_VERSION") - if version != "head" && compareVersion(version, "23.3") < 0 { - t.Skipf("Test skipped, BACKUP/RESTORE not production ready for %s version", version) - } - //t.Parallel() - r := require.New(t) - //CUSTOM backup create folder in each disk - r.NoError(dockerExec("clickhouse", "rm", "-rfv", "/var/lib/clickhouse/disks/backups_s3/backup/")) - runMainIntegrationScenario(t, "EMBEDDED_S3", "config-s3-embedded.yml") - //@TODO uncomment when resolve slow azure BACKUP/RESTORE https://github.com/ClickHouse/ClickHouse/issues/52088 - //r.NoError(dockerExec("clickhouse", "rm", "-rf", "/var/lib/clickhouse/disks/backups_azure/backup/")) - //runMainIntegrationScenario(t, "EMBEDDED_AZURE", "config-azblob-embedded.yml") - //@TODO think about how to implements embedded backup for s3_plain disks - //r.NoError(dockerExec("clickhouse", "rm", "-rf", "/var/lib/clickhouse/disks/backups_s3_plain/backup/")) - //runMainIntegrationScenario(t, "EMBEDDED_S3_PLAIN", "config-s3-plain-embedded.yml") -} +func TestRestoreMapping(t *testing.T) { + env, r := NewTestEnvironment(t) + env.connectWithWait(r, 500*time.Millisecond, 1*time.Second, 1*time.Minute) -func TestRestoreDatabaseMapping(t *testing.T) { - //t.Parallel() - r := require.New(t) - ch := &TestClickHouse{} - ch.connectWithWait(r, 500*time.Millisecond, 1*time.Second) - defer ch.chbackend.Close() checkRecordset := func(expectedRows int, expectedCount uint64, query string) { result := make([]struct { Count uint64 `ch:"count()"` }, 0) - r.NoError(ch.chbackend.Select(&result, query)) + r.NoError(env.ch.Select(&result, query)) r.Equal(expectedRows, len(result), "expect %d row", expectedRows) r.Equal(expectedCount, result[0].Count, "expect count=%d", expectedCount) } testBackupName := "test_restore_database_mapping" - databaseList := []string{"database1", "database2"} - fullCleanup(t, r, ch, []string{testBackupName}, []string{"local"}, databaseList, false, false, "config-database-mapping.yml") + databaseList := []string{"database1", "database-2"} + fullCleanup(t, r, env, []string{testBackupName}, []string{"local"}, databaseList, false, false, "config-database-mapping.yml") - ch.queryWithNoError(r, "CREATE DATABASE database1") - ch.queryWithNoError(r, "CREATE TABLE database1.t1 (dt DateTime, v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/database1/t1','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt") - ch.queryWithNoError(r, "CREATE TABLE database1.d1 AS database1.t1 ENGINE=Distributed('{cluster}',database1, t1)") + env.queryWithNoError(r, "CREATE DATABASE database1") + env.queryWithNoError(r, "CREATE TABLE database1.t1 (dt DateTime, v UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/database1/t1','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt") + env.queryWithNoError(r, "CREATE TABLE database1.d1 AS database1.t1 ENGINE=Distributed('{cluster}', 'database1', 't1')") if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.3") < 0 { - ch.queryWithNoError(r, "CREATE TABLE database1.t2 AS database1.t1 ENGINE=ReplicatedMergeTree('/clickhouse/tables/database1/t2','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt") + env.queryWithNoError(r, "CREATE TABLE database1.t2 AS database1.t1 ENGINE=ReplicatedMergeTree('/clickhouse/tables/database1/t2','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt") } else { - ch.queryWithNoError(r, "CREATE TABLE database1.t2 AS database1.t1 ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/{table}','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt") + env.queryWithNoError(r, "CREATE TABLE database1.t2 AS database1.t1 ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/{table}','{replica}') PARTITION BY toYYYYMM(dt) ORDER BY dt") } - ch.queryWithNoError(r, "CREATE MATERIALIZED VIEW database1.mv1 TO database1.t2 AS SELECT * FROM database1.t1") - ch.queryWithNoError(r, "CREATE VIEW database1.v1 AS SELECT * FROM database1.t1") - ch.queryWithNoError(r, "INSERT INTO database1.t1 SELECT '2022-01-01 00:00:00', number FROM numbers(10)") + env.queryWithNoError(r, "CREATE MATERIALIZED VIEW database1.mv1 TO database1.t2 AS SELECT * FROM database1.t1") + env.queryWithNoError(r, "CREATE VIEW database1.v1 AS SELECT * FROM database1.t1") + env.queryWithNoError(r, "INSERT INTO database1.t1 SELECT '2022-01-01 00:00:00', number FROM numbers(10)") - log.Info().Msg("Create backup") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "create", testBackupName)) + log.Debug().Msg("Create backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "create", testBackupName) - log.Info().Msg("Restore schema") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--schema", "--rm", "--restore-database-mapping", "database1:database2", "--tables", "database1.*", testBackupName)) + log.Debug().Msg("Restore schema") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--schema", "--rm", "--restore-database-mapping", "database1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,d1:d2,mv1:mv2,v1:v2", "--tables", "database1.*", testBackupName) - log.Info().Msg("Check result database1") - ch.queryWithNoError(r, "INSERT INTO database1.t1 SELECT '2023-01-01 00:00:00', number FROM numbers(10)") + log.Debug().Msg("Check result database1") + env.queryWithNoError(r, "INSERT INTO database1.t1 SELECT '2023-01-01 00:00:00', number FROM numbers(10)") checkRecordset(1, 20, "SELECT count() FROM database1.t1") checkRecordset(1, 20, "SELECT count() FROM database1.d1") checkRecordset(1, 20, "SELECT count() FROM database1.mv1") checkRecordset(1, 20, "SELECT count() FROM database1.v1") - log.Info().Msg("Drop database1") - r.NoError(ch.dropDatabase("database1")) + log.Debug().Msg("Drop database1") + r.NoError(env.dropDatabase("database1")) - log.Info().Msg("Restore data") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--data", "--restore-database-mapping", "database1:database2", "--tables", "database1.*", testBackupName)) + log.Debug().Msg("Restore data") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-database-mapping.yml", "restore", "--data", "--restore-database-mapping", "database1:database-2", "--restore-table-mapping", "t1:t3,t2:t4,d1:d2,mv1:mv2,v1:v2", "--tables", "database1.*", testBackupName) - log.Info().Msg("Check result database2") - checkRecordset(1, 10, "SELECT count() FROM database2.t1") - checkRecordset(1, 10, "SELECT count() FROM database2.d1") - checkRecordset(1, 10, "SELECT count() FROM database2.mv1") - checkRecordset(1, 10, "SELECT count() FROM database2.v1") + log.Debug().Msg("Check result database-2") + checkRecordset(1, 10, "SELECT count() FROM `database-2`.t3") + checkRecordset(1, 10, "SELECT count() FROM `database-2`.d2") + checkRecordset(1, 10, "SELECT count() FROM `database-2`.mv2") + checkRecordset(1, 10, "SELECT count() FROM `database-2`.v2") - log.Info().Msg("Check database1 not exists") - checkRecordset(1, 0, "SELECT count() FROM system.databases WHERE name='database1'") + log.Debug().Msg("Check database1 not exists") + checkRecordset(1, 0, "SELECT count() FROM system.databases WHERE name='database1' SETTINGS empty_result_for_aggregation_by_empty_set=0") - fullCleanup(t, r, ch, []string{testBackupName}, []string{"local"}, databaseList, true, true, "config-database-mapping.yml") + fullCleanup(t, r, env, []string{testBackupName}, []string{"local"}, databaseList, true, true, "config-database-mapping.yml") + env.Cleanup(t, r) } func TestMySQLMaterialized(t *testing.T) { - t.Skipf("Wait when fix DROP TABLE not supported by MaterializedMySQL, just attach will not help") + t.Skipf("Wait when fix DROP TABLE not supported by MaterializedMySQL, just attach will not help, https://github.com/ClickHouse/ClickHouse/issues/57543") if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.12") == -1 { t.Skipf("MaterializedMySQL doens't support for clickhouse version %s", os.Getenv("CLICKHOUSE_VERSION")) } - //t.Parallel() - r := require.New(t) - r.NoError(dockerExec("mysql", "mysql", "-u", "root", "--password=root", "-v", "-e", "CREATE DATABASE ch_mysql_repl")) - ch := &TestClickHouse{} - ch.connectWithWait(r, 500*time.Millisecond, 1*time.Second) - defer ch.chbackend.Close() + env, r := NewTestEnvironment(t) + env.DockerExecNoError(r, "mysql", "mysql", "-u", "root", "--password=root", "-v", "-e", "CREATE DATABASE ch_mysql_repl") + env.connectWithWait(r, 500*time.Millisecond, 1*time.Second, 1*time.Minute) engine := "MaterializedMySQL" if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.9") == -1 { engine = "MaterializeMySQL" } - ch.queryWithNoError(r, fmt.Sprintf("CREATE DATABASE ch_mysql_repl ENGINE=%s('mysql:3306','ch_mysql_repl','root','root')", engine)) - r.NoError(dockerExec("mysql", "mysql", "-u", "root", "--password=root", "-v", "-e", "CREATE TABLE ch_mysql_repl.t1 (id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, s VARCHAR(255)); INSERT INTO ch_mysql_repl.t1(s) VALUES('s1'),('s2'),('s3')")) + env.queryWithNoError(r, fmt.Sprintf("CREATE DATABASE ch_mysql_repl ENGINE=%s('mysql:3306','ch_mysql_repl','root','root')", engine)) + env.DockerExecNoError(r, "mysql", "mysql", "-u", "root", "--password=root", "-v", "-e", "CREATE TABLE ch_mysql_repl.t1 (id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, s VARCHAR(255)); INSERT INTO ch_mysql_repl.t1(s) VALUES('s1'),('s2'),('s3')") time.Sleep(1 * time.Second) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "test_mysql_materialized")) - ch.queryWithNoError(r, "DROP DATABASE ch_mysql_repl") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "test_mysql_materialized")) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "test_mysql_materialized") + env.queryWithNoError(r, "DROP DATABASE ch_mysql_repl") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "test_mysql_materialized") result := 0 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&result, "SELECT count() FROM ch_mysql_repl.t1")) + r.NoError(env.ch.SelectSingleRowNoCtx(&result, "SELECT count() FROM ch_mysql_repl.t1")) r.Equal(3, result, "expect count=3") - ch.queryWithNoError(r, "DROP DATABASE ch_mysql_repl") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_mysql_materialized")) + env.queryWithNoError(r, "DROP DATABASE ch_mysql_repl") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_mysql_materialized") + env.Cleanup(t, r) } func TestPostgreSQLMaterialized(t *testing.T) { - t.Skipf("Wait when fix https://github.com/ClickHouse/ClickHouse/issues/44250") - if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.11") == -1 { t.Skipf("MaterializedPostgreSQL doens't support for clickhouse version %s", os.Getenv("CLICKHOUSE_VERSION")) } - //t.Parallel() - r := require.New(t) - r.NoError(dockerExec("pgsql", "bash", "-ce", "echo 'CREATE DATABASE ch_pgsql_repl' | PGPASSWORD=root psql -v ON_ERROR_STOP=1 -U root")) - r.NoError(dockerExec("pgsql", "bash", "-ce", "echo \"CREATE TABLE t1 (id BIGINT PRIMARY KEY, s VARCHAR(255)); INSERT INTO t1(id, s) VALUES(1,'s1'),(2,'s2'),(3,'s3')\" | PGPASSWORD=root psql -v ON_ERROR_STOP=1 -U root -d ch_pgsql_repl")) - ch := &TestClickHouse{} - ch.connectWithWait(r, 500*time.Millisecond, 1*time.Second) - defer ch.chbackend.Close() + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "23.10") == -1 { + t.Skipf("Serial type, support in 23.10+, look https://github.com/ClickHouse/ClickHouse/issues/44250") + } + t.Skip("FREEZE don't support for MaterializedPostgreSQL, https://github.com/ClickHouse/ClickHouse/issues/32902") - ch.queryWithNoError(r, + env, r := NewTestEnvironment(t) + env.DockerExecNoError(r, "pgsql", "bash", "-ce", "echo 'CREATE DATABASE ch_pgsql_repl' | PGPASSWORD=root psql -v ON_ERROR_STOP=1 -U root") + env.DockerExecNoError(r, "pgsql", "bash", "-ce", "echo \"CREATE TABLE t1 (id BIGINT PRIMARY KEY, s VARCHAR(255)); INSERT INTO t1(id, s) VALUES(1,'s1'),(2,'s2'),(3,'s3')\" | PGPASSWORD=root psql -v ON_ERROR_STOP=1 -U root -d ch_pgsql_repl") + env.connectWithWait(r, 500*time.Millisecond, 1*time.Second, 1*time.Minute) + env.queryWithNoError(r, "CREATE DATABASE ch_pgsql_repl ENGINE=MaterializedPostgreSQL('pgsql:5432','ch_pgsql_repl','root','root') "+ - "SETTINGS materialized_postgresql_allow_automatic_update = 1, materialized_postgresql_schema = 'public'", + "SETTINGS materialized_postgresql_schema = 'public'", ) - time.Sleep(1 * time.Second) + // time to initial snapshot + count := uint64(0) + for { + err := env.ch.SelectSingleRowNoCtx(&count, "SELECT count() FROM system.tables WHERE database='ch_pgsql_repl'") + r.NoError(err) + if count > 0 { + break + } + log.Debug().Msgf("ch_pgsql_repl contains %d tables, wait 5 seconds", count) + time.Sleep(5 * time.Second) + } - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "test_pgsql_materialized")) - ch.queryWithNoError(r, "DROP DATABASE ch_pgsql_repl") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "test_pgsql_materialized")) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "create", "test_pgsql_materialized") + env.queryWithNoError(r, "DROP DATABASE ch_pgsql_repl") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "restore", "test_pgsql_materialized") result := 0 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&result, "SELECT count() FROM ch_pgsql_repl.t1")) + r.NoError(env.ch.SelectSingleRowNoCtx(&result, "SELECT count() FROM ch_pgsql_repl.t1")) r.Equal(3, result, "expect count=3") - ch.queryWithNoError(r, "DROP DATABASE ch_pgsql_repl") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_pgsql_materialized")) + env.queryWithNoError(r, "DROP DATABASE ch_pgsql_repl") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/config-s3.yml", "delete", "local", "test_pgsql_materialized") + env.Cleanup(t, r) } -func uploadSSHKeys(r *require.Assertions, container string) { - r.NoError(dockerCP("sftp/clickhouse-backup_rsa", container+":/id_rsa")) - r.NoError(dockerExec(container, "cp", "-vf", "/id_rsa", "/tmp/id_rsa")) - r.NoError(dockerExec(container, "chmod", "-v", "0600", "/tmp/id_rsa")) +func (env *TestEnvironment) uploadSSHKeys(r *require.Assertions, container string) { + r.NoError(env.DockerCP("sftp/clickhouse-backup_rsa", container+":/id_rsa")) + env.DockerExecNoError(r, container, "cp", "-vf", "/id_rsa", "/tmp/id_rsa") + env.DockerExecNoError(r, container, "chmod", "-v", "0600", "/tmp/id_rsa") - r.NoError(dockerCP("sftp/clickhouse-backup_rsa.pub", "sshd:/root/.ssh/authorized_keys")) - r.NoError(dockerExec("sshd", "chown", "-v", "root:root", "/root/.ssh/authorized_keys")) - r.NoError(dockerExec("sshd", "chmod", "-v", "0600", "/root/.ssh/authorized_keys")) + r.NoError(env.DockerCP("sftp/clickhouse-backup_rsa.pub", "sshd:/root/.ssh/authorized_keys")) + env.DockerExecNoError(r, "sshd", "chown", "-v", "root:root", "/root/.ssh/authorized_keys") + env.DockerExecNoError(r, "sshd", "chmod", "-v", "0600", "/root/.ssh/authorized_keys") } -func runMainIntegrationScenario(t *testing.T, remoteStorageType, backupConfig string) { +func (env *TestEnvironment) runMainIntegrationScenario(t *testing.T, remoteStorageType, backupConfig string) { var out string var err error - r := require.New(t) - ch := &TestClickHouse{} - ch.connectWithWait(r, 500*time.Millisecond, 1*time.Minute) - defer ch.chbackend.Close() - + env.connectWithWait(r, 500*time.Millisecond, 1500*time.Millisecond, 3*time.Minute) // test for specified partitions backup - testBackupSpecifiedPartitions(t, r, ch, remoteStorageType, backupConfig) + testBackupSpecifiedPartitions(t, r, env, remoteStorageType, backupConfig) // main test scenario - testBackupName := fmt.Sprintf("%s_full_%d", t.Name(), rand.Int()) + fullBackupName := fmt.Sprintf("%s_full_%d", t.Name(), rand.Int()) incrementBackupName := fmt.Sprintf("%s_increment_%d", t.Name(), rand.Int()) + incrementBackupName2 := fmt.Sprintf("%s_increment2_%d", t.Name(), rand.Int()) databaseList := []string{dbNameOrdinary, dbNameAtomic, dbNameMySQL, dbNamePostgreSQL, Issue331Atomic, Issue331Ordinary} tablesPattern := fmt.Sprintf("*_%s.*", t.Name()) - log.Info().Msg("Clean before start") - fullCleanup(t, r, ch, []string{testBackupName, incrementBackupName}, []string{"remote", "local"}, databaseList, false, false, backupConfig) + log.Debug().Msg("Clean before start") + fullCleanup(t, r, env, []string{fullBackupName, incrementBackupName}, []string{"remote", "local"}, databaseList, false, false, backupConfig) + + testData := generateTestData(t, r, env, remoteStorageType, defaultTestData) - r.NoError(dockerExec("minio", "mc", "ls", "local/clickhouse/disk_s3")) - testData := generateTestData(t, r, ch, remoteStorageType, defaultTestData) + log.Debug().Msg("Create backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create", "--tables", tablesPattern, fullBackupName) - r.NoError(dockerExec("minio", "mc", "ls", "local/clickhouse/disk_s3")) - log.Info().Msg("Create backup") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create", "--tables", tablesPattern, testBackupName)) - generateIncrementTestData(t, ch, r, defaultIncrementData) + incrementData := generateIncrementTestData(t, r, env, remoteStorageType, defaultIncrementData, 1) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create", "--tables", tablesPattern, incrementBackupName) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create", "--tables", tablesPattern, incrementBackupName)) + log.Debug().Msg("Upload full") + uploadCmd := fmt.Sprintf("%s_COMPRESSION_FORMAT=zstd CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/%s clickhouse-backup upload --resume %s", remoteStorageType, backupConfig, fullBackupName) + env.checkResumeAlreadyProcessed(uploadCmd, fullBackupName, "upload", r, remoteStorageType) - log.Info().Msg("Upload") - uploadCmd := fmt.Sprintf("%s_COMPRESSION_FORMAT=zstd CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/%s clickhouse-backup upload --resume %s", remoteStorageType, backupConfig, testBackupName) - checkResumeAlreadyProcessed(uploadCmd, testBackupName, "upload", r, remoteStorageType) + // https://github.com/Altinity/clickhouse-backup/pull/900 + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.8") >= 0 { + log.Debug().Msg("create --diff-from-remote backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create", "--diff-from-remote", fullBackupName, "--tables", tablesPattern, incrementBackupName2) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "upload", incrementBackupName2) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "remote", incrementBackupName2) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", incrementBackupName2) + } - //diffFrom := []string{"--diff-from", "--diff-from-remote"}[rand.Intn(2)] - diffFrom := "--diff-from-remote" - uploadCmd = fmt.Sprintf("clickhouse-backup -c /etc/clickhouse-backup/%s upload %s %s %s --resume", backupConfig, incrementBackupName, diffFrom, testBackupName) - checkResumeAlreadyProcessed(uploadCmd, incrementBackupName, "upload", r, remoteStorageType) + log.Debug().Msg("Upload increment") + uploadCmd = fmt.Sprintf("clickhouse-backup -c /etc/clickhouse-backup/%s upload %s --diff-from-remote %s --resume", backupConfig, incrementBackupName, fullBackupName) + env.checkResumeAlreadyProcessed(uploadCmd, incrementBackupName, "upload", r, remoteStorageType) backupDir := "/var/lib/clickhouse/backup" - if strings.HasPrefix(remoteStorageType, "EMBEDDED") { + if strings.HasPrefix(remoteStorageType, "EMBEDDED") && !strings.HasSuffix(remoteStorageType, "_URL") { backupDir = "/var/lib/clickhouse/disks/backups" + strings.ToLower(strings.TrimPrefix(remoteStorageType, "EMBEDDED")) } - out, err = dockerExecOut("clickhouse-backup", "bash", "-ce", "ls -lha "+backupDir+" | grep "+t.Name()) + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", "ls -lha "+backupDir+" | grep "+t.Name()) r.NoError(err) r.Equal(2, len(strings.Split(strings.Trim(out, " \t\r\n"), "\n")), "expect '2' backups exists in backup directory") - log.Info().Msg("Delete backup") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", testBackupName)) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", incrementBackupName)) - out, err = dockerExecOut("clickhouse-backup", "bash", "-ce", "ls -lha "+backupDir+" | grep "+t.Name()) + log.Debug().Msg("Delete backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", fullBackupName) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", incrementBackupName) + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", "ls -lha "+backupDir+" | grep "+t.Name()) r.NotNil(err) r.Equal("", strings.Trim(out, " \t\r\n"), "expect '0' backup exists in backup directory") - dropDatabasesFromTestDataDataSet(t, r, ch, databaseList) + dropDatabasesFromTestDataDataSet(t, r, env, databaseList) - log.Info().Msg("Download") - downloadCmd := fmt.Sprintf("clickhouse-backup -c /etc/clickhouse-backup/%s download --resume %s", backupConfig, testBackupName) - checkResumeAlreadyProcessed(downloadCmd, testBackupName, "download", r, remoteStorageType) + log.Debug().Msg("Download") + replaceStorageDiskNameForReBalance(r, env, remoteStorageType, false) + downloadCmd := fmt.Sprintf("clickhouse-backup -c /etc/clickhouse-backup/%s download --resume %s", backupConfig, fullBackupName) + env.checkResumeAlreadyProcessed(downloadCmd, fullBackupName, "download", r, remoteStorageType) - log.Info().Msg("Restore schema") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore", "--schema", testBackupName)) + log.Debug().Msg("Restore schema") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore", "--schema", fullBackupName) - log.Info().Msg("Restore data") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore", "--data", testBackupName)) + log.Debug().Msg("Restore data") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore", "--data", fullBackupName) - log.Info().Msg("Full restore with rm") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore", "--rm", testBackupName)) + log.Debug().Msg("Full restore with rm") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore", "--rm", fullBackupName) - log.Info().Msg("Check data") + log.Debug().Msg("Check data") for i := range testData { if testData[i].CheckDatabaseOnly { - r.NoError(ch.checkDatabaseEngine(t, testData[i])) + r.NoError(env.checkDatabaseEngine(t, testData[i])) } else { - if isTableSkip(ch, testData[i], true) { + if isTableSkip(env, testData[i], true) { continue } - r.NoError(ch.checkData(t, testData[i], r)) + r.NoError(env.checkData(t, r, testData[i])) } } // test increment - dropDatabasesFromTestDataDataSet(t, r, ch, databaseList) + dropDatabasesFromTestDataDataSet(t, r, env, databaseList) - log.Info().Msg("Delete backup") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", testBackupName)) + log.Debug().Msg("Delete backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", fullBackupName) - log.Info().Msg("Download increment") + log.Debug().Msg("Download increment") downloadCmd = fmt.Sprintf("clickhouse-backup -c /etc/clickhouse-backup/%s download --resume %s", backupConfig, incrementBackupName) - checkResumeAlreadyProcessed(downloadCmd, incrementBackupName, "download", r, remoteStorageType) + env.checkResumeAlreadyProcessed(downloadCmd, incrementBackupName, "download", r, remoteStorageType) - log.Info().Msg("Restore") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore", "--schema", "--data", incrementBackupName)) + log.Debug().Msg("Restore") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore", "--schema", "--data", incrementBackupName) - log.Info().Msg("Check increment data") + log.Debug().Msg("Check increment data") for i := range testData { testDataItem := testData[i] - if isTableSkip(ch, testDataItem, true) || testDataItem.IsDictionary { + if isTableSkip(env, testDataItem, true) || testDataItem.IsDictionary { continue } - for _, incrementDataItem := range defaultIncrementData { + for _, incrementDataItem := range incrementData { if testDataItem.Database == incrementDataItem.Database && testDataItem.Name == incrementDataItem.Name { testDataItem.Rows = append(testDataItem.Rows, incrementDataItem.Rows...) } } if testDataItem.CheckDatabaseOnly { - r.NoError(ch.checkDatabaseEngine(t, testDataItem)) + r.NoError(env.checkDatabaseEngine(t, testDataItem)) } else { - r.NoError(ch.checkData(t, testDataItem, r)) + r.NoError(env.checkData(t, r, testDataItem)) } - } // test end - log.Info().Msg("Clean after finish") - // CUSTOM and EMBEDDED download increment doesn't download full - if remoteStorageType == "CUSTOM" || strings.HasPrefix(remoteStorageType, "EMBEDDED") { - fullCleanup(t, r, ch, []string{incrementBackupName}, []string{"local"}, nil, true, false, backupConfig) - fullCleanup(t, r, ch, []string{testBackupName, incrementBackupName}, []string{"remote"}, databaseList, true, true, backupConfig) - } else { - fullCleanup(t, r, ch, []string{testBackupName, incrementBackupName}, []string{"remote", "local"}, databaseList, true, true, backupConfig) + log.Debug().Msg("Clean after finish") + // during download increment, partially downloaded full will also clean + fullCleanup(t, r, env, []string{incrementBackupName}, []string{"local"}, nil, true, false, backupConfig) + fullCleanup(t, r, env, []string{fullBackupName, incrementBackupName}, []string{"remote"}, databaseList, true, true, backupConfig) + replaceStorageDiskNameForReBalance(r, env, remoteStorageType, true) + env.checkObjectStorageIsEmpty(t, r, remoteStorageType) +} + +func (env *TestEnvironment) checkObjectStorageIsEmpty(t *testing.T, r *require.Assertions, remoteStorageType string) { + if remoteStorageType == "AZBLOB" || remoteStorageType == "AZBLOB_EMBEDDED_URL" { + t.Log("wait when resolve https://github.com/Azure/Azurite/issues/2362, todo try to use mysql as azurite storage") + /* + env.DockerExecNoError(r, "azure", "apk", "add", "jq") + checkBlobCollection := func(containerName string, expected string) { + out, err := env.DockerExecOut("azure", "sh", "-c", "jq '.collections[] | select(.name == \"$BLOBS_COLLECTION$\") | .data[] | select(.containerName == \""+containerName+"\") | .name' /data/__azurite_db_blob__.json") + r.NoError(err) + actual := strings.Trim(out, "\n\r\t ") + if expected != actual { + env.DockerExecNoError(r, "azure", "sh", "-c", "cat /data/__azurite_db_blob__.json | jq") + env.DockerExecNoError(r, "azure", "sh", "-c", "stat -c '%y' /data/__azurite_db_blob__.json") + env.DockerExecNoError(r, "azure", "sh", "-c", "cat /data/debug.log") + } + r.Equal(expected, actual) + } + // docker run --network=integration_clickhouse-backup -it --rm mcr.microsoft.com/azure-cli:latest + // export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure:10000/devstoreaccount1;" + // az storage blob list --container-name azure-disk + // az storage blob delete-batch --source azure-disk + // az storage blob list --container-name azure-disk + time.Sleep(15 * time.Second) + checkBlobCollection("azure-disk", "") + checkBlobCollection("container1", "") + checkBlobCollection("azure-backup-disk", "") + */ + } + checkRemoteDir := func(expected string, container string, cmd ...string) { + out, err := env.DockerExecOut(container, cmd...) + r.NoError(err) + r.Equal(expected, strings.Trim(out, "\r\n\t ")) + } + if remoteStorageType == "S3" || remoteStorageType == "S3_EMBEDDED_URL" { + checkRemoteDir("total 0", "minio", "bash", "-c", "ls -lh /bitnami/minio/data/clickhouse/") + } + if remoteStorageType == "SFTP" { + checkRemoteDir("total 0", "sshd", "bash", "-c", "ls -lh /root/") + } + if remoteStorageType == "FTP" { + if strings.Contains(os.Getenv("COMPOSE_FILE"), "advanced") { + checkRemoteDir("total 0", "ftp", "bash", "-c", "ls -lh /home/ftpusers/test_backup/backup/") + } else { + checkRemoteDir("total 0", "ftp", "bash", "-c", "ls -lh /home/vsftpd/test_backup/backup/") + } + } + if remoteStorageType == "GCS_EMULATOR" { + checkRemoteDir("total 0", "gcs", "sh", "-c", "ls -lh /data/altinity-qa-test/") + } +} + +func replaceStorageDiskNameForReBalance(r *require.Assertions, env *TestEnvironment, remoteStorageType string, isRebalanced bool) { + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "23.3") < 0 { + return + } + if remoteStorageType != "S3" && remoteStorageType != "GCS" && remoteStorageType != "AZBLOB" { + return + } + oldDisk := "disk_" + strings.ToLower(remoteStorageType) + newDisk := oldDisk + "_rebalanced" + if isRebalanced { + oldDisk = "disk_" + strings.ToLower(remoteStorageType) + "_rebalanced" + newDisk = strings.TrimSuffix(oldDisk, "_rebalanced") + } + fileNames := []string{"storage_configuration_" + strings.ToLower(remoteStorageType) + ".xml"} + if remoteStorageType == "S3" { + fileNames = append(fileNames, "storage_configuration_encrypted_"+strings.ToLower(remoteStorageType)+".xml") + } + for _, fileName := range fileNames { + origFile := "/etc/clickhouse-server/config.d/" + fileName + dstFile := "/var/lib/clickhouse/" + fileName + sedCmd := fmt.Sprintf("s/<%s>/<%s>/g; s/<\\/%s>/<\\/%s>/g; s/%s<\\/disk>/%s<\\/disk>/g", oldDisk, newDisk, oldDisk, newDisk, oldDisk, newDisk) + env.DockerExecNoError(r, "clickhouse", "sed", "-i", sedCmd, origFile) + env.DockerExecNoError(r, "clickhouse", "cp", "-vf", origFile, dstFile) } + if isRebalanced { + env.DockerExecNoError(r, "clickhouse", "bash", "-xc", "cp -aflv -t /var/lib/clickhouse/disks/"+newDisk+"/ /var/lib/clickhouse/disks/"+oldDisk+"/*") + env.DockerExecNoError(r, "clickhouse", "rm", "-rf", "/var/lib/clickhouse/disks/"+oldDisk+"") + } + env.ch.Close() + r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker", append(env.GetDefaultComposeCommand(), "restart", "clickhouse")...)) + env.connectWithWait(r, 3*time.Second, 1500*time.Millisecond, 3*time.Minute) } -func testBackupSpecifiedPartitions(t *testing.T, r *require.Assertions, ch *TestClickHouse, remoteStorageType string, backupConfig string) { - log.Info().Msg("testBackupSpecifiedPartitions started") +func testBackupSpecifiedPartitions(t *testing.T, r *require.Assertions, env *TestEnvironment, remoteStorageType string, backupConfig string) { + log.Debug().Msg("testBackupSpecifiedPartitions started") var err error var out string var result, expectedCount uint64 @@ -1938,84 +2620,132 @@ func testBackupSpecifiedPartitions(t *testing.T, r *require.Assertions, ch *Test fullBackupName := fmt.Sprintf("full_backup_%d", rand.Int()) dbName := "test_partitions_" + t.Name() // Create and fill tables - ch.queryWithNoError(r, "CREATE DATABASE IF NOT EXISTS "+dbName) - ch.queryWithNoError(r, "DROP TABLE IF EXISTS "+dbName+".t1") - ch.queryWithNoError(r, "DROP TABLE IF EXISTS "+dbName+".t2") - ch.queryWithNoError(r, "CREATE TABLE "+dbName+".t1 (dt Date, v UInt64) ENGINE=MergeTree() PARTITION BY toYYYYMMDD(dt) ORDER BY dt") - ch.queryWithNoError(r, "CREATE TABLE "+dbName+".t2 (dt String, v UInt64) ENGINE=MergeTree() PARTITION BY dt ORDER BY dt") + env.queryWithNoError(r, "CREATE DATABASE IF NOT EXISTS "+dbName) + env.queryWithNoError(r, "DROP TABLE IF EXISTS "+dbName+".t1") + env.queryWithNoError(r, "DROP TABLE IF EXISTS "+dbName+".t2") + env.queryWithNoError(r, "CREATE TABLE "+dbName+".t1 (dt Date, category Int64, v UInt64) ENGINE=MergeTree() PARTITION BY (category, toYYYYMMDD(dt)) ORDER BY dt") + env.queryWithNoError(r, "CREATE TABLE "+dbName+".t2 (dt String, category Int64, v UInt64) ENGINE=MergeTree() PARTITION BY (category, dt) ORDER BY dt") for _, dt := range []string{"2022-01-01", "2022-01-02", "2022-01-03", "2022-01-04"} { - ch.queryWithNoError(r, fmt.Sprintf("INSERT INTO "+dbName+".t1 SELECT '%s', number FROM numbers(10)", dt)) - ch.queryWithNoError(r, fmt.Sprintf("INSERT INTO "+dbName+".t2 SELECT '%s', number FROM numbers(10)", dt)) + env.queryWithNoError(r, fmt.Sprintf("INSERT INTO "+dbName+".t1(dt, v) SELECT '%s', number FROM numbers(10)", dt)) + env.queryWithNoError(r, fmt.Sprintf("INSERT INTO "+dbName+".t2(dt, v) SELECT '%s', number FROM numbers(10)", dt)) } - // check create_remote full > download + partitions > delete local > download > restore --partitions > restore - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create_remote", "--tables="+dbName+".t*", fullBackupName)) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", fullBackupName)) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "download", "--partitions=('2022-01-02'),('2022-01-03')", fullBackupName)) + // check create_remote full > download + partitions > restore --data --partitions > delete local > download > restore --partitions > restore + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create_remote", "--tables="+dbName+".t*", fullBackupName) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", fullBackupName) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "download", "--partitions="+dbName+".t?:(0,'2022-01-02'),(0,'2022-01-03')", fullBackupName) fullBackupDir := "/var/lib/clickhouse/backup/" + fullBackupName + "/shadow/" + dbName + "/t?/default/" + // embedded storage with embedded disks contains object disk files and will download additional data parts if strings.HasPrefix(remoteStorageType, "EMBEDDED") { fullBackupDir = "/var/lib/clickhouse/disks/backups" + strings.ToLower(strings.TrimPrefix(remoteStorageType, "EMBEDDED")) + "/" + fullBackupName + "/data/" + dbName + "/t?" } - out, err = dockerExecOut("clickhouse-backup", "bash", "-c", "ls -la "+fullBackupDir+" | wc -l") + // embedded storage without embedded disks doesn't contain `shadow` and contain only `metadata` + if strings.HasPrefix(remoteStorageType, "EMBEDDED") && strings.HasSuffix(remoteStorageType, "_URL") { + fullBackupDir = "/var/lib/clickhouse/backup/" + fullBackupName + "/metadata/" + dbName + "/t?.json" + } + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-c", "ls -la "+fullBackupDir+" | wc -l") r.NoError(err) expectedLines := "13" // custom storage doesn't support --partitions for upload / download now - // embedded storage contain hardLink files and will download additional data parts + // embedded storage with embedded disks contain hardLink files and will download additional data parts if remoteStorageType == "CUSTOM" || strings.HasPrefix(remoteStorageType, "EMBEDDED") { expectedLines = "17" } + // embedded storage without embedded disks doesn't contain `shadow` and contain only `metadata` + if strings.HasPrefix(remoteStorageType, "EMBEDDED") && strings.HasSuffix(remoteStorageType, "_URL") { + expectedLines = "2" + } r.Equal(expectedLines, strings.Trim(out, "\r\n\t ")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", fullBackupName)) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "download", fullBackupName)) + checkRestoredDataWithPartitions := func(expectedCount uint64) { + result = 0 + r.NoError(env.ch.SelectSingleRowNoCtx(&result, "SELECT sum(c) FROM (SELECT count() AS c FROM "+dbName+".t1 UNION ALL SELECT count() AS c FROM "+dbName+".t2)")) + r.Equal(expectedCount, result, "expect count=%d", expectedCount) + } + + if remoteStorageType == "FTP" && !strings.Contains(backupConfig, "old") { + // during DROP PARTITION, we create empty covered part, and cant restore via ATTACH TABLE properly, https://github.com/Altinity/clickhouse-backup/issues/756 + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", "clickhouse-backup -c /etc/clickhouse-backup/"+backupConfig+" restore --data --partitions=\"(0,'2022-01-02'),(0,'2022-01-03')\" "+fullBackupName) + r.Error(err) + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", "CLICKHOUSE_RESTORE_AS_ATTACH=0 clickhouse-backup -c /etc/clickhouse-backup/"+backupConfig+" restore --data --partitions=\"(0,'2022-01-02'),(0,'2022-01-03')\" "+fullBackupName) + } else { + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-ce", "clickhouse-backup -c /etc/clickhouse-backup/"+backupConfig+" restore --data --partitions=\"(0,'2022-01-02'),(0,'2022-01-03')\" "+fullBackupName) + } + log.Debug().Msg(out) + r.NoError(err) + r.Contains(out, "DROP PARTITION") + // we just replace data in exists table + checkRestoredDataWithPartitions(80) + + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", fullBackupName) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "download", fullBackupName) + expectedLines = "17" fullBackupDir = "/var/lib/clickhouse/backup/" + fullBackupName + "/shadow/" + dbName + "/t?/default/" + // embedded storage with embedded disks contains hardLink files and will download additional data parts if strings.HasPrefix(remoteStorageType, "EMBEDDED") { fullBackupDir = "/var/lib/clickhouse/disks/backups" + strings.ToLower(strings.TrimPrefix(remoteStorageType, "EMBEDDED")) + "/" + fullBackupName + "/data/" + dbName + "/t?" } - out, err = dockerExecOut("clickhouse-backup", "bash", "-c", "ls -la "+fullBackupDir+"| wc -l") + // embedded storage without embedded disks doesn't contain `shadow` and contain only `metadata` + if strings.HasPrefix(remoteStorageType, "EMBEDDED") && strings.HasSuffix(remoteStorageType, "_URL") { + fullBackupDir = "/var/lib/clickhouse/backup/" + fullBackupName + "/metadata/" + dbName + "/t?.json" + expectedLines = "2" + } + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-c", "ls -la "+fullBackupDir+"| wc -l") r.NoError(err) - r.Equal("17", strings.Trim(out, "\r\n\t ")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore", "--partitions=('2022-01-02'),('2022-01-03')", fullBackupName)) - result = 0 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&result, "SELECT sum(c) FROM (SELECT count() AS c FROM "+dbName+".t1 UNION ALL SELECT count() AS c FROM "+dbName+".t2)")) - expectedCount = 40 - r.Equal(expectedCount, result, fmt.Sprintf("expect count=%d", expectedCount)) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore", fullBackupName)) - result = 0 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&result, "SELECT sum(c) FROM (SELECT count() AS c FROM "+dbName+".t1 UNION ALL SELECT count() AS c FROM "+dbName+".t2)")) - r.Equal(uint64(80), result, "expect count=80") - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "remote", fullBackupName)) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", fullBackupName)) + r.Equal(expectedLines, strings.Trim(out, "\r\n\t ")) + + out, err = env.DockerExecOut("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore", "--partitions=(0,'2022-01-02'),(0,'2022-01-03')", fullBackupName) + r.NoError(err) + r.NotContains(out, "DROP PARTITION") + checkRestoredDataWithPartitions(40) + + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore", fullBackupName) + checkRestoredDataWithPartitions(80) + + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "remote", fullBackupName) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", fullBackupName) // check create + partitions - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create", "--tables="+dbName+".t1", "--partitions=20220102,20220103", partitionBackupName)) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create", "--tables="+dbName+".t1", "--partitions=(0,'2022-01-02'),(0,'2022-01-03')", partitionBackupName) + expectedLines = "5" partitionBackupDir := "/var/lib/clickhouse/backup/" + partitionBackupName + "/shadow/" + dbName + "/t1/default/" - if strings.HasPrefix(remoteStorageType, "EMBEDDED") { + if strings.HasPrefix(remoteStorageType, "EMBEDDED") && !strings.HasSuffix(remoteStorageType, "_URL") { partitionBackupDir = "/var/lib/clickhouse/disks/backups" + strings.ToLower(strings.TrimPrefix(remoteStorageType, "EMBEDDED")) + "/" + partitionBackupName + "/data/" + dbName + "/t1" } - out, err = dockerExecOut("clickhouse-backup", "bash", "-c", "ls -la "+partitionBackupDir+"| wc -l") + //embedded backup without a disk has only local metadata + if strings.HasPrefix(remoteStorageType, "EMBEDDED") && strings.HasSuffix(remoteStorageType, "_URL") { + partitionBackupDir = "/var/lib/clickhouse/backup/" + partitionBackupName + "/metadata/" + dbName + "/t?.json" + expectedLines = "1" + } + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-c", "ls -la "+partitionBackupDir+"| wc -l") r.NoError(err) - r.Equal("5", strings.Trim(out, "\r\n\t ")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", partitionBackupName)) + r.Equal(expectedLines, strings.Trim(out, "\r\n\t ")) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", partitionBackupName) // check create > upload + partitions - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create", "--tables="+dbName+".t1", partitionBackupName)) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "create", "--tables="+dbName+".t1", partitionBackupName) partitionBackupDir = "/var/lib/clickhouse/backup/" + partitionBackupName + "/shadow/" + dbName + "/t1/default/" - if strings.HasPrefix(remoteStorageType, "EMBEDDED") { + expectedLines = "7" + if strings.HasPrefix(remoteStorageType, "EMBEDDED") && !strings.HasSuffix(remoteStorageType, "_URL") { partitionBackupDir = "/var/lib/clickhouse/disks/backups" + strings.ToLower(strings.TrimPrefix(remoteStorageType, "EMBEDDED")) + "/" + partitionBackupName + "/data/" + dbName + "/t1" } - out, err = dockerExecOut("clickhouse-backup", "bash", "-c", "ls -la "+partitionBackupDir+" | wc -l") + //embedded backup without a disk has only local metadata + if strings.HasPrefix(remoteStorageType, "EMBEDDED") && strings.HasSuffix(remoteStorageType, "_URL") { + partitionBackupDir = "/var/lib/clickhouse/backup/" + partitionBackupName + "/metadata/" + dbName + "/t?.json" + expectedLines = "1" + } + out, err = env.DockerExecOut("clickhouse-backup", "bash", "-c", "ls -la "+partitionBackupDir+" | wc -l") r.NoError(err) - r.Equal("7", strings.Trim(out, "\r\n\t ")) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "upload", "--tables="+dbName+".t1", "--partitions=20220102,20220103", partitionBackupName)) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", partitionBackupName)) + r.Equal(expectedLines, strings.Trim(out, "\r\n\t ")) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "upload", "--tables="+dbName+".t1", "--partitions=0-20220102,0-20220103", partitionBackupName) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", partitionBackupName) // restore partial uploaded - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore_remote", partitionBackupName)) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "restore_remote", partitionBackupName) // Check partial restored t1 result = 0 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&result, "SELECT count() FROM "+dbName+".t1")) + r.NoError(env.ch.SelectSingleRowNoCtx(&result, "SELECT count() FROM "+dbName+".t1")) expectedCount = 20 // custom and embedded doesn't support --partitions in upload and download @@ -2026,7 +2756,7 @@ func testBackupSpecifiedPartitions(t *testing.T, r *require.Assertions, ch *Test // Check only selected partitions restored result = 0 - r.NoError(ch.chbackend.SelectSingleRowNoCtx(&result, "SELECT count() FROM "+dbName+".t1 WHERE dt NOT IN ('2022-01-02','2022-01-03')")) + r.NoError(env.ch.SelectSingleRowNoCtx(&result, "SELECT count() FROM "+dbName+".t1 WHERE dt NOT IN ('2022-01-02','2022-01-03')")) expectedCount = 0 // custom and embedded doesn't support --partitions in upload and download if remoteStorageType == "CUSTOM" || strings.HasPrefix(remoteStorageType, "EMBEDDED") { @@ -2035,44 +2765,44 @@ func testBackupSpecifiedPartitions(t *testing.T, r *require.Assertions, ch *Test r.Equal(expectedCount, result, "expect count=0") // DELETE backup. - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "remote", partitionBackupName)) - r.NoError(dockerExec("clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", partitionBackupName)) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "remote", partitionBackupName) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "-c", "/etc/clickhouse-backup/"+backupConfig, "delete", "local", partitionBackupName) - if err = ch.dropDatabase(dbName); err != nil { + if err = env.dropDatabase(dbName); err != nil { t.Fatal(err) } - log.Info().Msg("testBackupSpecifiedPartitions finish") + log.Debug().Msg("testBackupSpecifiedPartitions finish") } -func checkResumeAlreadyProcessed(backupCmd, testBackupName, resumeKind string, r *require.Assertions, remoteStorageType string) { +func (env *TestEnvironment) checkResumeAlreadyProcessed(backupCmd, testBackupName, resumeKind string, r *require.Assertions, remoteStorageType string) { // backupCmd = fmt.Sprintf("%s & PID=$!; sleep 0.7; kill -9 $PID; cat /var/lib/clickhouse/backup/%s/upload.state; sleep 0.3; %s", backupCmd, testBackupName, backupCmd) if remoteStorageType == "CUSTOM" || strings.HasPrefix(remoteStorageType, "EMBEDDED") { backupCmd = strings.Replace(backupCmd, "--resume", "", 1) } else { backupCmd = fmt.Sprintf("%s; cat /var/lib/clickhouse/backup/%s/%s.state; %s", backupCmd, testBackupName, resumeKind, backupCmd) } - out, err := dockerExecOut("clickhouse-backup", "bash", "-xce", backupCmd) - log.Info().Msg(out) + out, err := env.DockerExecOut("clickhouse-backup", "bash", "-xce", backupCmd) + log.Debug().Msg(out) r.NoError(err) if strings.Contains(backupCmd, "--resume") { r.Contains(out, "already processed") } } -func fullCleanup(t *testing.T, r *require.Assertions, ch *TestClickHouse, backupNames, backupTypes, databaseList []string, checkDeleteErr, checkDropErr bool, backupConfig string) { +func fullCleanup(t *testing.T, r *require.Assertions, env *TestEnvironment, backupNames, backupTypes, databaseList []string, checkDeleteErr, checkDropErr bool, backupConfig string) { for _, backupName := range backupNames { for _, backupType := range backupTypes { - err := dockerExec("clickhouse-backup", "bash", "-xce", "clickhouse-backup -c /etc/clickhouse-backup/"+backupConfig+" delete "+backupType+" "+backupName) + err := env.DockerExec("clickhouse-backup", "bash", "-xce", "clickhouse-backup -c /etc/clickhouse-backup/"+backupConfig+" delete "+backupType+" "+backupName) if checkDeleteErr { - r.NoError(err) + r.NoError(err, "checkDeleteErr delete %s %s error: %v", err, backupType, backupName) } } } - otherBackupList, err := dockerExecOut("clickhouse", "ls", "-1", "/var/lib/clickhouse/backup/*"+t.Name()+"*") + otherBackupList, err := env.DockerExecOut("clickhouse", "ls", "-1", "/var/lib/clickhouse/backup/*"+t.Name()+"*") if err == nil { for _, backupName := range strings.Split(otherBackupList, "\n") { if backupName != "" { - err := dockerExec("clickhouse-backup", "bash", "-xce", "clickhouse-backup -c /etc/clickhouse-backup/"+backupConfig+" delete local "+backupName) + err := env.DockerExec("clickhouse-backup", "bash", "-xce", "clickhouse-backup -c /etc/clickhouse-backup/"+backupConfig+" delete local "+backupName) if checkDropErr { r.NoError(err) } @@ -2080,35 +2810,36 @@ func fullCleanup(t *testing.T, r *require.Assertions, ch *TestClickHouse, backup } } - dropDatabasesFromTestDataDataSet(t, r, ch, databaseList) + dropDatabasesFromTestDataDataSet(t, r, env, databaseList) } -func generateTestData(t *testing.T, r *require.Assertions, ch *TestClickHouse, remoteStorageType string, testData []TestDataStruct) []TestDataStruct { - log.Info().Msgf("Generate test data %s with _%s suffix", remoteStorageType, t.Name()) - testData = generateTestDataWithDifferentStoragePolicy(remoteStorageType, testData) +func generateTestData(t *testing.T, r *require.Assertions, env *TestEnvironment, remoteStorageType string, testData []TestDataStruct) []TestDataStruct { + log.Debug().Msgf("Generate test data %s with _%s suffix", remoteStorageType, t.Name()) + testData = generateTestDataWithDifferentStoragePolicy(remoteStorageType, 0, 5, testData) for _, data := range testData { - if isTableSkip(ch, data, false) { + if isTableSkip(env, data, false) { continue } - r.NoError(ch.createTestSchema(t, data, remoteStorageType)) + r.NoError(env.createTestSchema(t, data, remoteStorageType)) } for _, data := range testData { - if isTableSkip(ch, data, false) { + if isTableSkip(env, data, false) { continue } - r.NoError(ch.createTestData(t, data)) + r.NoError(env.createTestData(t, data)) } return testData } -func generateTestDataWithDifferentStoragePolicy(remoteStorageType string, testData []TestDataStruct) []TestDataStruct { +func generateTestDataWithDifferentStoragePolicy(remoteStorageType string, offset, rowsCount int, testData []TestDataStruct) []TestDataStruct { + log.Debug().Msgf("generateTestDataWithDifferentStoragePolicy remoteStorageType=%s", remoteStorageType) for databaseName, databaseEngine := range map[string]string{dbNameOrdinary: "Ordinary", dbNameAtomic: "Atomic"} { testDataWithStoragePolicy := TestDataStruct{ Database: databaseName, DatabaseEngine: databaseEngine, Rows: func() []map[string]interface{} { - result := make([]map[string]interface{}, 100) - for i := 0; i < 100; i++ { - result[i] = map[string]interface{}{"id": uint64(i)} + result := make([]map[string]interface{}, rowsCount) + for i := 0; i < rowsCount; i++ { + result[i] = map[string]interface{}{"id": uint64(i + offset)} } return result }(), @@ -2117,9 +2848,10 @@ func generateTestDataWithDifferentStoragePolicy(remoteStorageType string, testDa } addTestDataIfNotExists := func() { found := false - for _, data := range testData { + for i, data := range testData { if data.Name == testDataWithStoragePolicy.Name && data.Database == testDataWithStoragePolicy.Database { found = true + testData[i].Rows = testDataWithStoragePolicy.Rows break } } @@ -2128,86 +2860,87 @@ func generateTestDataWithDifferentStoragePolicy(remoteStorageType string, testDa } } //s3 disks support after 21.8 - if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.8") >= 0 && remoteStorageType == "S3" { + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.8") >= 0 && strings.Contains(remoteStorageType, "S3") { testDataWithStoragePolicy.Name = "test_s3" - testDataWithStoragePolicy.Schema = "(id UInt64) Engine=MergeTree ORDER BY id SETTINGS storage_policy = 's3_only'" + testDataWithStoragePolicy.Schema = "(id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY id PARTITION BY id SETTINGS storage_policy = 's3_only'" addTestDataIfNotExists() } //encrypted disks support after 21.10 if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.10") >= 0 { testDataWithStoragePolicy.Name = "test_hdd3_encrypted" - testDataWithStoragePolicy.Schema = "(id UInt64) Engine=MergeTree ORDER BY id SETTINGS storage_policy = 'hdd3_only_encrypted'" + testDataWithStoragePolicy.Schema = "(id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY id PARTITION BY id SETTINGS storage_policy = 'hdd3_only_encrypted'" addTestDataIfNotExists() } //encrypted s3 disks support after 21.12 - if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 && remoteStorageType == "S3" { + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") >= 0 && strings.Contains(remoteStorageType, "S3") { testDataWithStoragePolicy.Name = "test_s3_encrypted" - testDataWithStoragePolicy.Schema = "(id UInt64) Engine=MergeTree ORDER BY id SETTINGS storage_policy = 's3_only_encrypted'" + testDataWithStoragePolicy.Schema = "(id UInt64) Engine=MergeTree ORDER BY id PARTITION BY id SETTINGS storage_policy = 's3_only_encrypted'" + // @todo wait when fix https://github.com/ClickHouse/ClickHouse/issues/58247 + //if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "23.12") >= 0 { + // testDataWithStoragePolicy.Schema = "(id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY id PARTITION BY id SETTINGS storage_policy = 's3_only_encrypted'" + //} addTestDataIfNotExists() } //gcs over s3 support added in 22.6 if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "22.6") >= 0 && remoteStorageType == "GCS" && os.Getenv("QA_GCS_OVER_S3_BUCKET") != "" { testDataWithStoragePolicy.Name = "test_gcs" - testDataWithStoragePolicy.Schema = "(id UInt64) Engine=MergeTree ORDER BY id SETTINGS storage_policy = 'gcs_only'" + testDataWithStoragePolicy.Schema = "(id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY id PARTITION BY id SETTINGS storage_policy = 'gcs_only'" addTestDataIfNotExists() } //check azure_blob_storage only in 23.3+ (added in 22.1) - if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "23.3") >= 0 && remoteStorageType == "AZBLOB" { + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "23.3") >= 0 && strings.Contains(remoteStorageType, "AZBLOB") { testDataWithStoragePolicy.Name = "test_azure" - testDataWithStoragePolicy.Schema = "(id UInt64) Engine=MergeTree ORDER BY id SETTINGS storage_policy = 'azure_only'" + testDataWithStoragePolicy.Schema = "(id UInt64) Engine=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/{database}/{table}','{replica}') ORDER BY id PARTITION BY id SETTINGS storage_policy = 'azure_only'" addTestDataIfNotExists() } } return testData } -func generateIncrementTestData(t *testing.T, ch *TestClickHouse, r *require.Assertions, incrementData []TestDataStruct) { - log.Info().Msg("Generate increment test data") +func generateIncrementTestData(t *testing.T, r *require.Assertions, ch *TestEnvironment, remoteStorageType string, incrementData []TestDataStruct, incrementNumber int) []TestDataStruct { + log.Debug().Msgf("Generate increment test data for %s", remoteStorageType) + incrementData = generateTestDataWithDifferentStoragePolicy(remoteStorageType, 5*incrementNumber, 5, incrementData) for _, data := range incrementData { if isTableSkip(ch, data, false) { continue } r.NoError(ch.createTestData(t, data)) } + return incrementData } -func dropDatabasesFromTestDataDataSet(t *testing.T, r *require.Assertions, ch *TestClickHouse, databaseList []string) { - log.Info().Msg("Drop all databases") +func dropDatabasesFromTestDataDataSet(t *testing.T, r *require.Assertions, ch *TestEnvironment, databaseList []string) { + log.Debug().Msg("Drop all databases") for _, db := range databaseList { - r.NoError(ch.dropDatabase(db + "_" + t.Name())) + db = db + "_" + t.Name() + r.NoError(ch.dropDatabase(db)) } } -const apiBackupNumber = 5 - -type TestClickHouse struct { - chbackend *clickhouse.ClickHouse -} - -func (ch *TestClickHouse) connectWithWait(r *require.Assertions, sleepBefore, timeOut time.Duration) { +func (env *TestEnvironment) connectWithWait(r *require.Assertions, sleepBefore, pollInterval, timeOut time.Duration) { time.Sleep(sleepBefore) for i := 1; i < 11; i++ { - err := ch.connect(timeOut.String()) + err := env.connect(timeOut.String()) if i == 10 { - r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker", "logs", "clickhouse")) - out, dockerErr := dockerExecOut("clickhouse", "clickhouse client", "--echo", "-q", "'SELECT version()'") + r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker", append(env.GetDefaultComposeCommand(), "logs", "clickhouse")...)) + out, dockerErr := env.DockerExecOut("clickhouse", "clickhouse", "client", "--echo", "-q", "'SELECT version()'") r.NoError(dockerErr) - log.Warn().Msg(out) + log.Debug().Msg(out) r.NoError(err) } if err != nil { r.NoError(utils.ExecCmd(context.Background(), 180*time.Second, "docker", "ps", "-a")) - if out, dockerErr := dockerExecOut("clickhouse", "clickhouse client", "--echo", "-q", "SELECT version()"); dockerErr == nil { - log.Warn().Msg(out) + if out, dockerErr := env.DockerExecOut("clickhouse", "clickhouse", "client", "--echo", "-q", "SELECT version()"); dockerErr == nil { + log.Debug().Msg(out) } else { log.Info().Msg(out) } - log.Warn().Msgf("clickhouse not ready %v, wait %v seconds", err, (time.Duration(i) * timeOut).Seconds()) - time.Sleep(time.Duration(i) * timeOut) + log.Warn().Msgf("clickhouse not ready %v, wait %v seconds", err, (pollInterval).Seconds()) + time.Sleep(pollInterval) } else { - if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.8") == 1 { + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.8") > 0 { var count uint64 - err = ch.chbackend.SelectSingleRowNoCtx(&count, "SELECT count() FROM mysql('mysql:3306','mysql','user','root','root')") + err = env.ch.SelectSingleRowNoCtx(&count, "SELECT count() FROM mysql('mysql:3306','mysql','user','root','root')") if err == nil { break } else { @@ -2221,17 +2954,30 @@ func (ch *TestClickHouse) connectWithWait(r *require.Assertions, sleepBefore, ti } } -func (ch *TestClickHouse) connect(timeOut string) error { - ch.chbackend = &clickhouse.ClickHouse{ +func (env *TestEnvironment) connect(timeOut string) error { + portOut, err := utils.ExecCmdOut(context.Background(), 10*time.Second, "docker", append(env.GetDefaultComposeCommand(), "port", "clickhouse", "9000")...) + if err != nil { + log.Error().Msg(portOut) + log.Fatal().Msgf("can't get port for clickhouse: %v", err) + } + hostAndPort := strings.Split(strings.Trim(portOut, " \r\n\t"), ":") + if len(hostAndPort) < 1 { + log.Error().Msg(portOut) + log.Fatal().Msgf("invalid port for clickhouse: %v", err) + } + port, err := strconv.Atoi(hostAndPort[1]) + if err != nil { + return err + } + env.ch = &clickhouse.ClickHouse{ Config: &config.ClickHouseConfig{ - Host: "127.0.0.1", - Port: 9000, + Host: hostAndPort[0], + Port: uint(port), Timeout: timeOut, }, } - var err error for i := 0; i < 3; i++ { - err = ch.chbackend.Connect() + err = env.ch.Connect() if err == nil { return nil } else { @@ -2243,7 +2989,7 @@ func (ch *TestClickHouse) connect(timeOut string) error { var mergeTreeOldSyntax = regexp.MustCompile(`(?m)MergeTree\(([^,]+),([\w\s,)(]+),(\s*\d+\s*)\)`) -func (ch *TestClickHouse) createTestSchema(t *testing.T, data TestDataStruct, remoteStorageType string) error { +func (env *TestEnvironment) createTestSchema(t *testing.T, data TestDataStruct, remoteStorageType string) error { origDatabase := data.Database origName := data.Name if !data.IsFunction { @@ -2251,11 +2997,11 @@ func (ch *TestClickHouse) createTestSchema(t *testing.T, data TestDataStruct, re data.Name = data.Name + "_" + t.Name() // 20.8 doesn't respect DROP TABLE ... NO DELAY, so Atomic works but --rm is not applicable if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.8") > 0 { - if err := ch.chbackend.CreateDatabaseWithEngine(data.Database, data.DatabaseEngine, "cluster"); err != nil { + if err := env.ch.CreateDatabaseWithEngine(data.Database, data.DatabaseEngine, "cluster"); err != nil { return err } } else { - if err := ch.chbackend.CreateDatabase(data.Database, "cluster"); err != nil { + if err := env.ch.CreateDatabase(data.Database, "cluster"); err != nil { return err } } @@ -2289,7 +3035,7 @@ func (ch *TestClickHouse) createTestSchema(t *testing.T, data TestDataStruct, re // old 1.x clickhouse versions doesn't contains {table} and {database} macros if strings.Contains(createSQL, "{table}") || strings.Contains(createSQL, "{database}") { var isMacrosExists uint64 - if err := ch.chbackend.SelectSingleRowNoCtx(&isMacrosExists, "SELECT count() FROM system.functions WHERE name='getMacro'"); err != nil { + if err := env.ch.SelectSingleRowNoCtx(&isMacrosExists, "SELECT count() FROM system.functions WHERE name='getMacro'"); err != nil { return err } if isMacrosExists == 0 { @@ -2320,59 +3066,64 @@ func (ch *TestClickHouse) createTestSchema(t *testing.T, data TestDataStruct, re createSQL = strings.NewReplacer("."+origName, "."+data.Name, "`"+origName+"`", "`"+data.Name+"`", "'"+origName+"'", "'"+data.Name+"'").Replace(createSQL) } createSQL = strings.Replace(createSQL, "{test}", t.Name(), -1) - err := ch.chbackend.CreateTable( + err := env.ch.CreateTable( clickhouse.Table{ Database: data.Database, Name: data.Name, }, createSQL, - false, false, "", 0, + false, false, "", 0, "/var/lib/clickhouse", ) return err } -func (ch *TestClickHouse) createTestData(t *testing.T, data TestDataStruct) error { +func (env *TestEnvironment) createTestData(t *testing.T, data TestDataStruct) error { data.Database = data.Database + "_" + t.Name() data.Name = data.Name + "_" + t.Name() if data.SkipInsert || data.CheckDatabaseOnly { return nil } insertSQL := fmt.Sprintf("INSERT INTO `%s`.`%s`", data.Database, data.Name) - - batch, err := ch.chbackend.GetConn().PrepareBatch(context.Background(), insertSQL) + log.Debug().Msg(insertSQL) + batch, err := env.ch.GetConn().PrepareBatch(context.Background(), insertSQL) if err != nil { - return err + return fmt.Errorf("createTestData PrepareBatch(%s) error: %v", insertSQL, err) } for _, row := range data.Rows { insertData := make([]interface{}, len(data.Fields)) + log.Debug().Msgf("VALUES %v", row) for idx, field := range data.Fields { insertData[idx] = row[field] } if err = batch.Append(insertData...); err != nil { - return err + return fmt.Errorf("createTestData batch.Append(%#v) error: %v", insertData, err) } } - return batch.Send() + err = batch.Send() + if err != nil { + return fmt.Errorf("createTestData batch.Send(%s) error: %v", insertSQL, err) + } + return err } -func (ch *TestClickHouse) dropDatabase(database string) (err error) { +func (env *TestEnvironment) dropDatabase(database string) (err error) { var isAtomic bool dropDatabaseSQL := fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", database) - if isAtomic, err = ch.chbackend.IsAtomic(database); isAtomic { + if isAtomic, err = env.ch.IsAtomic(database); isAtomic { dropDatabaseSQL += " SYNC" } else if err != nil { return err } - return ch.chbackend.Query(dropDatabaseSQL) + return env.ch.Query(dropDatabaseSQL) } -func (ch *TestClickHouse) checkData(t *testing.T, data TestDataStruct, r *require.Assertions) error { +func (env *TestEnvironment) checkData(t *testing.T, r *require.Assertions, data TestDataStruct) error { assert.NotNil(t, data.Rows) data.Database += "_" + t.Name() data.Name += "_" + t.Name() - log.Info().Msgf("Check '%d' rows in '%s.%s'\n", len(data.Rows), data.Database, data.Name) + log.Debug().Msgf("Check '%d' rows in '%s.%s'\n", len(data.Rows), data.Database, data.Name) selectSQL := fmt.Sprintf("SELECT * FROM `%s`.`%s` ORDER BY `%s`", data.Database, data.Name, strings.Replace(data.OrderBy, "{test}", t.Name(), -1)) if data.IsFunction && compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.12") == -1 { @@ -2382,7 +3133,7 @@ func (ch *TestClickHouse) checkData(t *testing.T, data TestDataStruct, r *requir selectSQL = fmt.Sprintf("SELECT %s(number, number+1) AS test_result FROM numbers(%d)", data.Name, len(data.Rows)) } log.Debug().Msg(selectSQL) - rows, err := ch.chbackend.GetConn().Query(context.Background(), selectSQL) + rows, err := env.ch.GetConn().Query(context.Background(), selectSQL) if err != nil { return err } @@ -2430,14 +3181,14 @@ func (ch *TestClickHouse) checkData(t *testing.T, data TestDataStruct, r *requir return nil } -func (ch *TestClickHouse) checkDatabaseEngine(t *testing.T, data TestDataStruct) error { +func (env *TestEnvironment) checkDatabaseEngine(t *testing.T, data TestDataStruct) error { data.Database += "_" + t.Name() if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.8") <= 0 { return nil } selectSQL := fmt.Sprintf("SELECT engine FROM system.databases WHERE name='%s'", data.Database) var engine string - if err := ch.chbackend.SelectSingleRowNoCtx(&engine, selectSQL); err != nil { + if err := env.ch.SelectSingleRowNoCtx(&engine, selectSQL); err != nil { return err } assert.True( @@ -2447,35 +3198,83 @@ func (ch *TestClickHouse) checkDatabaseEngine(t *testing.T, data TestDataStruct) return nil } -func (ch *TestClickHouse) queryWithNoError(r *require.Assertions, query string, args ...interface{}) { - err := ch.chbackend.Query(query, args...) +func (env *TestEnvironment) queryWithNoError(r *require.Assertions, query string, args ...interface{}) { + err := env.ch.Query(query, args...) + if err != nil { + log.Error().Err(err).Msgf("queryWithNoError(%s) error", query) + } r.NoError(err) } -var dockerExecTimeout = 180 * time.Second +var dockerExecTimeout = 600 * time.Second + +func (env *TestEnvironment) DockerExecBackgroundNoError(r *require.Assertions, container string, cmd ...string) { + out, err := env.DockerExecBackgroundOut(container, cmd...) + r.NoError(err, "%s\n\n%s\n[ERROR]\n%v", strings.Join(append(append(env.GetDefaultComposeCommand(), "exec", "-d", container), cmd...), " "), out, err) +} -func dockerExec(container string, cmd ...string) error { - out, err := dockerExecOut(container, cmd...) - log.Info().Msg(out) +func (env *TestEnvironment) DockerExecBackground(container string, cmd ...string) error { + out, err := env.DockerExecBackgroundOut(container, cmd...) + log.Debug().Msg(out) return err } -func dockerExecOut(container string, cmd ...string) (string, error) { - dcmd := []string{"exec", container} +func (env *TestEnvironment) DockerExecBackgroundOut(container string, cmd ...string) (string, error) { + dcmd := append(env.GetDefaultComposeCommand(), "exec", "-d", container) dcmd = append(dcmd, cmd...) return utils.ExecCmdOut(context.Background(), dockerExecTimeout, "docker", dcmd...) } -func dockerCP(src, dst string) error { +func (env *TestEnvironment) GetDefaultComposeCommand() []string { + return []string{"compose", "-f", path.Join(os.Getenv("CUR_DIR"), os.Getenv("COMPOSE_FILE")), "--progress", "plain", "--project-name", env.ProjectName} +} + +func (env *TestEnvironment) GetExecDockerCommand(container string) []string { + return []string{"exec", fmt.Sprintf("%s-%s-1", env.ProjectName, container)} +} + +func (env *TestEnvironment) DockerExecNoError(r *require.Assertions, container string, cmd ...string) { + out, err := env.DockerExecOut(container, cmd...) + if err == nil { + log.Debug().Msg(out) + } + r.NoError(err, "%s\n\n%s\n[ERROR]\n%v", strings.Join(append(env.GetExecDockerCommand(container), cmd...), " "), out, err) +} + +func (env *TestEnvironment) DockerExec(container string, cmd ...string) error { + out, err := env.DockerExecOut(container, cmd...) + log.Debug().Msg(out) + return err +} + +func (env *TestEnvironment) DockerExecOut(container string, cmd ...string) (string, error) { + dcmd := append(env.GetExecDockerCommand(container), cmd...) + return utils.ExecCmdOut(context.Background(), dockerExecTimeout, "docker", dcmd...) +} + +func (env *TestEnvironment) DockerCP(src, dst string) error { ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) - dcmd := []string{"cp", src, dst} - log.Info().Msgf("docker %s", strings.Join(dcmd, " ")) + dcmd := append(env.GetDefaultComposeCommand(), "cp", src, dst) + + log.Debug().Msgf("docker %s", strings.Join(dcmd, " ")) out, err := exec.CommandContext(ctx, "docker", dcmd...).CombinedOutput() - log.Info().Msg(string(out)) + log.Debug().Msgf(string(out)) cancel() return err } +func (env *TestEnvironment) InstallDebIfNotExists(r *require.Assertions, container string, pkgs ...string) { + err := env.DockerExec( + container, + "bash", "-xec", + fmt.Sprintf( + "export DEBIAN_FRONTEND=noniteractive; if [[ '%d' != $(dpkg -l | grep -c -E \"%s\" ) ]]; then rm -fv /etc/apt/sources.list.d/clickhouse.list; find /etc/apt/ -type f -name *.list -exec sed -i 's/ru.archive.ubuntu.com/archive.ubuntu.com/g' {} +; apt-get -y update; apt-get install --no-install-recommends -y %s; fi", + len(pkgs), "^ii\\s+"+strings.Join(pkgs, "|^ii\\s+"), strings.Join(pkgs, " "), + ), + ) + r.NoError(err) +} + func toDate(s string) time.Time { result, _ := time.Parse("2006-01-02", s) return result @@ -2486,7 +3285,7 @@ func toTS(s string) time.Time { return result } -func isTableSkip(ch *TestClickHouse, data TestDataStruct, dataExists bool) bool { +func isTableSkip(ch *TestEnvironment, data TestDataStruct, dataExists bool) bool { if strings.Contains(data.DatabaseEngine, "PostgreSQL") && compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.3") <= 0 { return true } @@ -2498,10 +3297,10 @@ func isTableSkip(ch *TestClickHouse, data TestDataStruct, dataExists bool) bool "SELECT engine FROM system.tables WHERE name='%s' AND database='%s'", data.Name, data.Database, ) - _ = ch.chbackend.Select(&dictEngines, dictSQL) + _ = ch.ch.Select(&dictEngines, dictSQL) return len(dictEngines) == 0 } - return os.Getenv("COMPOSE_FILE") == "docker-compose.yml" && (strings.Contains(data.Name, "jbod_table") || data.IsDictionary) + return os.Getenv("COMPOSE_FILE") == "docker-compose.yml" && (strings.Contains(data.Name, "jbod#$_table") || data.IsDictionary) } func compareVersion(v1, v2 string) int { @@ -2526,14 +3325,3 @@ func isTestShouldSkip(envName string) bool { isSkip, _ := map[string]bool{"": true, "0": true, "false": true, "False": true, "1": false, "True": false, "true": false}[os.Getenv(envName)] return isSkip } - -func installDebIfNotExists(r *require.Assertions, container string, pkgs ...string) { - r.NoError(dockerExec( - container, - "bash", "-xec", - fmt.Sprintf( - "export DEBIAN_FRONTEND=noniteractive; if [[ '%d' != $(dpkg -l | grep -c -E \"%s\" ) ]]; then rm -fv /etc/apt/sources.list.d/clickhouse.list; find /etc/apt/ -type f -name *.list -exec sed -i 's/ru.archive.ubuntu.com/archive.ubuntu.com/g' {} +; apt-get -y update; apt-get install --no-install-recommends -y %s; fi", - len(pkgs), "^ii\\s+"+strings.Join(pkgs, "|^ii\\s+"), strings.Join(pkgs, " "), - ), - )) -} diff --git a/test/integration/kopia/checksum_parser.sh b/test/integration/kopia/checksum_parser.sh new file mode 100755 index 00000000..b0ff1405 --- /dev/null +++ b/test/integration/kopia/checksum_parser.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -eo pipefail +command -v clickhouse-compressor || exit 0 +CHECKSUM_FILE=$1 +ACTION=$2 +ACTION_DIR=$3 +if [[ "ENC" == $(dd if="${CHECKSUM_FILE}" bs=1 skip="0" count="3" 2>/dev/null) ]]; then + echo "ENCRYPTED FILES don't supported" + exit 0 +fi +FORMAT_VERSION=$(head -n +1 "${CHECKSUM_FILE}" | sed 's/checksums format version: //g') + +log() { printf '%s\n' "$*"; } +error() { log "ERROR: $*" >&2; } +fatal() { error "$@"; exit 1; } + +# appends a command to a trap +# +# - 1st arg: code to add +# - remaining args: names of traps to modify +# +trap_add() { + trap_add_cmd=$1; shift || fatal "${FUNCNAME} usage error" + for trap_add_name in "$@"; do + trap -- "$( + # helper fn to get existing trap command from output + # of trap -p + extract_trap_cmd() { printf '%s\n' "$3"; } + # print existing trap command with newline + eval "extract_trap_cmd $(trap -p "${trap_add_name}")" + # print the new trap command + printf '%s\n' "${trap_add_cmd}" + )" "${trap_add_name}" \ + || fatal "unable to add to trap ${trap_add_name}" + done +} + +function checksums_body_cmd { + if [[ "4" == "${FORMAT_VERSION}" ]]; then + tail -n +2 "${CHECKSUM_FILE}" | clickhouse-compressor -d + else + tail -n +2 "${CHECKSUM_FILE}" + fi +} + + +declare -g CURRENT_OFFSET=1 +CURRENT_OFFSET_FIFO=$(mktemp -u) # Generate a unique temporary file name +touch $CURRENT_OFFSET_FIFO +trap_add 'rm -f $CURRENT_OFFSET_FIFO' EXIT + +function read_uvarint { + readonly MaxVarintLen64=10 + readonly const0x80=$(printf "%d" 0x80) + readonly const0x7f=$(printf "%d" 0x7f) + local x=0 + local s=0 + + for ((i=0; i $CURRENT_OFFSET_FIFO + if [ -z "$byte_value" ]; then + if [ $i -gt 0 ]; then + fatal "Error: unexpected end of file" >&2 + fi + echo "$x" + return + fi + + if [ $byte_value -lt $const0x80 ]; then + if [ $i -eq $((MaxVarintLen64-1)) ] && [ "$byte_value" -gt 1 ]; then + fatal "Error: overflow" >&2 + fi + x=$((x | (byte_value << s))) + echo "$x" + return + fi + + x=$((x | ((byte_value & $const0x7f) << s))) + s=$((s + 7)) + done + + echo "$x" >&2 + fatal "Error: overflow" >&2 +} + +TEMP_CHECKSUM_BODY=$(mktemp) +trap_add 'rm -f "${TEMP_CHECKSUM_BODY}"' EXIT + +checksums_body_cmd > "${TEMP_CHECKSUM_BODY}" + +ITEMS_COUNT=$(hexdump -v -e '/1 "%u\n"' "${TEMP_CHECKSUM_BODY}" | read_uvarint) +read CURRENT_OFFSET < $CURRENT_OFFSET_FIFO + +for ((i=1; i<=$ITEMS_COUNT; i++)); do + NAME_LENGTH=$(tail -c +$CURRENT_OFFSET "${TEMP_CHECKSUM_BODY}" | hexdump -v -e '/1 "%u\n"' | read_uvarint) + read CURRENT_OFFSET < $CURRENT_OFFSET_FIFO + + NAME=$(dd if="${TEMP_CHECKSUM_BODY}" bs=1 skip="$((CURRENT_OFFSET-1))" count="${NAME_LENGTH}" 2>/dev/null) + ((CURRENT_OFFSET += NAME_LENGTH)) + + FILE_SIZE=$(tail -c +$CURRENT_OFFSET "${TEMP_CHECKSUM_BODY}" | hexdump -v -e '/1 "%u\n"' | read_uvarint) + read CURRENT_OFFSET < $CURRENT_OFFSET_FIFO + + FILE_HASH=$(dd if="${TEMP_CHECKSUM_BODY}" bs=1 skip="$((CURRENT_OFFSET-1))" count="16" 2>/dev/null | xxd -ps -c 32) + ((CURRENT_OFFSET += 16)) + + IS_COMPRESSED=$(dd if="${TEMP_CHECKSUM_BODY}" bs=1 skip="$((CURRENT_OFFSET-1))" count="1" 2>/dev/null | xxd -p) + ((CURRENT_OFFSET += 1)) + + if [ "00" != "$IS_COMPRESSED" ]; then + UNCOMPRESSED_SIZE=$(tail -c +$CURRENT_OFFSET "${TEMP_CHECKSUM_BODY}" | hexdump -v -e '/1 "%u\n"' | read_uvarint) + read CURRENT_OFFSET < $CURRENT_OFFSET_FIFO + + UNCOMPRESSED_HASH=$(dd if="${TEMP_CHECKSUM_BODY}" bs=1 skip="$((CURRENT_OFFSET-1))" count="16" 2>/dev/null | xxd -ps -c 32) + ((CURRENT_OFFSET += 16)) + fi + if [[ "upload" == "$ACTION" ]]; then + # echo "$(dirname ${CHECKSUM_FILE})/${NAME} -> ${ACTION_DIR}/${FILE_HASH}" + cp -fl "$(dirname ${CHECKSUM_FILE})/${NAME}" "${ACTION_DIR}/${FILE_HASH}" + rm "$(dirname ${CHECKSUM_FILE})/${NAME}" + elif [[ "download" == "$ACTION" ]]; then + # echo "${ACTION_DIR}/${FILE_HASH} -> $(dirname ${CHECKSUM_FILE})/${NAME}" + cp -fl "${ACTION_DIR}/${FILE_HASH}" "$(dirname ${CHECKSUM_FILE})/${NAME}" + else + echo "${ACTION_DIR}/${FILE_HASH} <-> $(dirname ${CHECKSUM_FILE})/${NAME}" + fi +done diff --git a/test/integration/kopia/download.sh b/test/integration/kopia/download.sh index f95b1943..e2f78bcc 100755 --- a/test/integration/kopia/download.sh +++ b/test/integration/kopia/download.sh @@ -6,5 +6,13 @@ BACKUP_NAME=$1 ${CUR_DIR}/list.sh | grep "${BACKUP_NAME}" | while IFS= read -r line; do SNAPSHOT_ID=$(echo "${line}" | jq -r -c -M .snapshot_id) SNAPSHOT_PATH=$(echo "${line}" | jq -r -c -M .snapshot_path) - kopia restore "${SNAPSHOT_ID}" --skip-existing ${SNAPSHOT_PATH} + kopia restore --parallel=$(nproc) "${SNAPSHOT_ID}" --skip-existing ${SNAPSHOT_PATH} + LOCAL_BACKUP_DIR="$(dirname ${SNAPSHOT_PATH})/${BACKUP_NAME}" + rm -rf "${LOCAL_BACKUP_DIR}" + find "${SNAPSHOT_PATH}" -type f -name checksums.txt | parallel -j $(nproc) "${CUR_DIR}/checksum_parser.sh" {} "download" "${SNAPSHOT_PATH}" + # need separately `rm` cause hash file can contains multiple the same files in different parts + find ${SNAPSHOT_PATH} -maxdepth 1 -type f -regex '.*/[a-z0-9]\{32\}.*' | while read HASH_FILE; do + rm "${SNAPSHOT_PATH}/${HASH_FILE}" + done || true + mv -f "${SNAPSHOT_PATH}" "${LOCAL_BACKUP_DIR}" done diff --git a/test/integration/kopia/init.sh b/test/integration/kopia/init.sh index 7af33791..11a06583 100755 --- a/test/integration/kopia/init.sh +++ b/test/integration/kopia/init.sh @@ -1,14 +1,14 @@ CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" export KOPIA_PASSWORD_FILE="${CUR_DIR}/password" export KOPIA_S3_BUCKET=clickhouse -export KOPIA_S3_PATH=/clickhouse/kopia/cluster_name/shard_number/ +export KOPIA_S3_PATH=/kopia/cluster_name/shard_number/ export KOPIA_S3_ENDPOINT=minio:9000 -export AWS_ACCESS_KEY_ID=access-key -export AWS_SECRET_ACCESS_KEY=it-is-my-super-secret-key +export AWS_ACCESS_KEY_ID=access_key +export AWS_SECRET_ACCESS_KEY=it_is_my_super_secret_key export KOPIA_KEEP_LAST=7 export KOPIA_PASSWORD=kopia-repo-password export KOPIA_CHECK_FOR_UPDATES=false export CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-custom-kopia.yml export CLICKHOUSE_PARAMS="--host '$(yq '.clickhouse.host' ${CLICKHOUSE_BACKUP_CONFIG})' --port '$(yq '.clickhouse.port' ${CLICKHOUSE_BACKUP_CONFIG})' --user '$(yq '.clickhouse.username' ${CLICKHOUSE_BACKUP_CONFIG})' --password '$(yq '.clickhouse.password' ${CLICKHOUSE_BACKUP_CONFIG})'" -kopia repository connect s3 --endpoint=${KOPIA_S3_ENDPOINT} --disable-tls --bucket=${KOPIA_S3_BUCKET} --prefix=${KOPIA_S3_PATH} --access-key=${AWS_ACCESS_KEY_ID} --secret-access-key=${AWS_SECRET_ACCESS_KEY} || kopia repository create s3 --endpoint=${KOPIA_S3_ENDPOINT} --disable-tls --bucket=${KOPIA_S3_BUCKET} --prefix=${KOPIA_S3_PATH} --access-key=${AWS_ACCESS_KEY_ID} --secret-access-key=${AWS_SECRET_ACCESS_KEY} -kopia policy set --global --keep-latest=${KOPIA_KEEP_LAST} \ No newline at end of file +kopia repository connect s3 --endpoint=${KOPIA_S3_ENDPOINT} --disable-tls-verification --bucket=${KOPIA_S3_BUCKET} --prefix=${KOPIA_S3_PATH} --access-key=${AWS_ACCESS_KEY_ID} --secret-access-key=${AWS_SECRET_ACCESS_KEY} || kopia repository create s3 --endpoint=${KOPIA_S3_ENDPOINT} --disable-tls-verification --bucket=${KOPIA_S3_BUCKET} --prefix=${KOPIA_S3_PATH} --access-key=${AWS_ACCESS_KEY_ID} --secret-access-key=${AWS_SECRET_ACCESS_KEY} +kopia policy set --global --keep-latest=${KOPIA_KEEP_LAST} diff --git a/test/integration/kopia/list.sh b/test/integration/kopia/list.sh index ac02e0a0..4bb4ce2c 100755 --- a/test/integration/kopia/list.sh +++ b/test/integration/kopia/list.sh @@ -3,4 +3,4 @@ set +x set -euo pipefail CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" source "${CUR_DIR}/init.sh" &>/dev/null -kopia snapshot list --storage-stats --json | jq -c -M '.[] | {"snapshot_id": .id, "snapshot_path": .source.path, "backup_name": .tags["tag:backup_name"], "creation_date": .endTime, "data_size": .storageStats.newData.packedContentBytes, "metadata_size": 0 }' \ No newline at end of file +kopia snapshot list --storage-stats --json | jq -c -M '.[] | {"snapshot_id": .id, "snapshot_path": .source.path, "backup_name": .tags["tag:backup_name"], "creation_date": .startTime, "upload_date": .endTime, "data_size": .storageStats.newData.packedContentBytes, "metadata_size": 0 }' diff --git a/test/integration/kopia/upload.sh b/test/integration/kopia/upload.sh index 9b442dd3..e7f6b8d1 100755 --- a/test/integration/kopia/upload.sh +++ b/test/integration/kopia/upload.sh @@ -7,14 +7,25 @@ DIFF_FROM_REMOTE=${2:-} DIFF_FROM_REMOTE_CMD="" LOCAL_PATHS=$(eval "clickhouse client $CLICKHOUSE_PARAMS -q \"SELECT concat(trim(TRAILING '/' FROM path),'/backup/','${BACKUP_NAME}') FROM system.disks FORMAT TSVRaw\" | awk '{printf(\"%s \",\$0)} END { printf \"\n\" }' || clickhouse client $CLICKHOUSE_PARAMS -q \"SELECT concat(replaceRegexpOne(metadata_path,'/metadata.*$',''),'/backup/','${BACKUP_NAME}') FROM system.tables WHERE database = 'system' AND metadata_path!='' LIMIT 1 FORMAT TSVRaw\" | awk '{printf(\"%s \",\$0)} END { printf \"\n\" }'") if [[ "" != "${DIFF_FROM_REMOTE}" ]]; then -# DIFF_FROM_REMOTE_CMD="--parent ${DIFF_FROM_REMOTE}" - DIFF_FROM_REMOTE_CMD="" + DIFF_FROM_REMOTE_CMD="--parent ${DIFF_FROM_REMOTE}" fi SNAPSHOT_SOURCES="" for dir in $(echo "${LOCAL_PATHS}"); do - echo "${dir}" if [[ -d "${dir}" ]]; then - SNAPSHOT_SOURCES="${dir} ${SNAPSHOT_SOURCES}" + UPLOAD_DIR="$(dirname "${dir}")/latest" + rm -rf "${UPLOAD_DIR}" + cp -rl "${dir}" "${UPLOAD_DIR}" + find "${UPLOAD_DIR}" -type f -name checksums.txt | parallel -j $(nproc) "${CUR_DIR}/checksum_parser.sh" {} "upload" "${UPLOAD_DIR}" + SNAPSHOT_SOURCES="${UPLOAD_DIR} ${SNAPSHOT_SOURCES}" fi done -kopia snapshot create $DIFF_FROM_REMOTE_CMD --fail-fast --tags="backup_name:${BACKUP_NAME}" $SNAPSHOT_SOURCES + +kopia snapshot create $DIFF_FROM_REMOTE_CMD --parallel=$(nproc) --fail-fast --tags="backup_name:${BACKUP_NAME}" $SNAPSHOT_SOURCES + +for dir in $(echo "${LOCAL_PATHS}"); do + if [[ -d "${dir}" ]]; then + UPLOAD_DIR="$(dirname "${dir}")/latest" + rm -rfv "${UPLOAD_DIR}" + fi +done + diff --git a/test/integration/minio.crt b/test/integration/minio.crt new file mode 100644 index 00000000..2cfcdc14 --- /dev/null +++ b/test/integration/minio.crt @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIIB5jCCAY2gAwIBAgIRAKVariXVkoJXt4Em34DQ1IgwCgYIKoZIzj0EAwIwNzEc +MBoGA1UEChMTQ2VydGdlbiBEZXZlbG9wbWVudDEXMBUGA1UECwwOc2xhY2hAU0xB +Q0gtUEMwHhcNMjQwODAxMTAzOTM2WhcNMjUwODAxMTAzOTM2WjA3MRwwGgYDVQQK +ExNDZXJ0Z2VuIERldmVsb3BtZW50MRcwFQYDVQQLDA5zbGFjaEBTTEFDSC1QQzBZ +MBMGByqGSM49AgEGCCqGSM49AwEHA0IABPOQnV4JH/ypTyV6dJUAbdre4aQ9Yk8S +ae1B+uqPurv+esBX64Bs8gA25dvxlNo9Uqr30vfju9YTK5IK1V5/aeCjejB4MA4G +A1UdDwEB/wQEAwICpDATBgNVHSUEDDAKBggrBgEFBQcDATAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBT+hDmJjxhMaVGgJDXKTQ/XUf7RPDAhBgNVHREEGjAYggls +b2NhbGhvc3SCBW1pbmlvhwR/AAABMAoGCCqGSM49BAMCA0cAMEQCIH95FDuWBAY9 +UV4ohQMF32EvshqQFtz3aWDz5ulRgGW1AiAaxLLTiX0bHt1zb2yz9rz1g42ry/OG +qyAQZM8OgmBKfA== +-----END CERTIFICATE----- diff --git a/test/integration/minio.key b/test/integration/minio.key new file mode 100644 index 00000000..409e55f9 --- /dev/null +++ b/test/integration/minio.key @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgknJEz0YS+f0dTHR2 +BHyY965LP6iENcxvD3hDh44p0BqhRANCAATzkJ1eCR/8qU8lenSVAG3a3uGkPWJP +EmntQfrqj7q7/nrAV+uAbPIANuXb8ZTaPVKq99L347vWEyuSCtVef2ng +-----END PRIVATE KEY----- diff --git a/test/integration/minio_nodelete.sh b/test/integration/minio_nodelete.sh index 369fba89..2180deab 100755 --- a/test/integration/minio_nodelete.sh +++ b/test/integration/minio_nodelete.sh @@ -30,5 +30,5 @@ EOF ) mc admin policy attach local nodelete --user nodelete -mc alias set nodelete http://localhost:9000 nodelete nodelete_password +mc alias set nodelete https://localhost:9000 nodelete nodelete_password mc alias list diff --git a/test/integration/restic/delete.sh b/test/integration/restic/delete.sh index 04449094..762f3400 100755 --- a/test/integration/restic/delete.sh +++ b/test/integration/restic/delete.sh @@ -3,4 +3,4 @@ set -xeuo pipefail CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" source "${CUR_DIR}/init.sh" BACKUP_NAME=$1 -restic forget --tag "${BACKUP_NAME}" --prune +restic forget --insecure-tls --tag "${BACKUP_NAME}" --prune --unsafe-allow-remove-all diff --git a/test/integration/restic/download.sh b/test/integration/restic/download.sh index c6a728bf..4a592db2 100755 --- a/test/integration/restic/download.sh +++ b/test/integration/restic/download.sh @@ -5,4 +5,4 @@ source "${CUR_DIR}/init.sh" BACKUP_NAME=$1 BACKUP_INFO=$(${CUR_DIR}/list.sh | grep "${BACKUP_NAME}") SNAPSHOT_ID=$(echo "${BACKUP_INFO}" | jq -r -c -M '.snapshot_id') -restic restore --tag "${BACKUP_NAME}" --target / "${SNAPSHOT_ID}" \ No newline at end of file +restic restore --insecure-tls --tag "${BACKUP_NAME}" --target / "${SNAPSHOT_ID}" \ No newline at end of file diff --git a/test/integration/restic/init.sh b/test/integration/restic/init.sh index 7859fbf7..0feebacd 100755 --- a/test/integration/restic/init.sh +++ b/test/integration/restic/init.sh @@ -1,9 +1,9 @@ CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" export RESTIC_PASSWORD_FILE="${CUR_DIR}/password" -export RESTIC_REPOSITORY=s3:http://minio:9000/clickhouse/restic/cluster_name/shard_number -export AWS_ACCESS_KEY_ID=access-key -export AWS_SECRET_ACCESS_KEY=it-is-my-super-secret-key +export RESTIC_REPOSITORY=s3:https://minio:9000/clickhouse/restic/cluster_name/shard_number +export AWS_ACCESS_KEY_ID=access_key +export AWS_SECRET_ACCESS_KEY=it_is_my_super_secret_key export RESTIC_KEEP_LAST=7 export CLICKHOUSE_BACKUP_CONFIG=/etc/clickhouse-backup/config-custom-restic.yml export CLICKHOUSE_PARAMS="--host '$(yq '.clickhouse.host' ${CLICKHOUSE_BACKUP_CONFIG})' --port '$(yq '.clickhouse.port' ${CLICKHOUSE_BACKUP_CONFIG})' --user '$(yq '.clickhouse.username' ${CLICKHOUSE_BACKUP_CONFIG})' --password '$(yq '.clickhouse.password' ${CLICKHOUSE_BACKUP_CONFIG})'" -restic cat config > /dev/null || restic init \ No newline at end of file +restic cat config --insecure-tls > /dev/null || restic init --insecure-tls diff --git a/test/integration/restic/list.sh b/test/integration/restic/list.sh index c4893bb1..98f7f0d2 100755 --- a/test/integration/restic/list.sh +++ b/test/integration/restic/list.sh @@ -4,9 +4,9 @@ set -euo pipefail CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" source "${CUR_DIR}/init.sh" rm -rf /tmp/restic_list_full.json -restic snapshots --json | jq -c -M '.[] | {"snapshot_id": .short_id, "backup_name": .tags[0], "creation_date": .time }' > /tmp/restic_list.json +restic snapshots --insecure-tls --json | jq -c -M '.[] | {"snapshot_id": .short_id, "backup_name": .tags[0], "creation_date": .time, "upload_date": .time }' > /tmp/restic_list.json jq -c -r -M --slurp '.[].snapshot_id' /tmp/restic_list.json | while IFS= read -r snapshot_id ; do - jq -c -M -s 'add' <(grep ${snapshot_id} /tmp/restic_list.json) <(restic stats --json ${snapshot_id}) >> /tmp/restic_list_full.json + jq -c -M -s 'add' <(grep ${snapshot_id} /tmp/restic_list.json) <(restic stats --insecure-tls --json ${snapshot_id}) >> /tmp/restic_list_full.json done cat /tmp/restic_list_full.json | jq -c -M --slurp '.[] | .data_size = .total_size | .metadata_size = 0' -set -x \ No newline at end of file +set -x diff --git a/test/integration/restic/upload.sh b/test/integration/restic/upload.sh index 5d430234..9fc3866c 100755 --- a/test/integration/restic/upload.sh +++ b/test/integration/restic/upload.sh @@ -10,5 +10,5 @@ if [[ "" != "${DIFF_FROM_REMOTE}" ]]; then DIFF_FROM_REMOTE=$(${CUR_DIR}/list.sh | grep "${DIFF_FROM_REMOTE}" | jq -r -c '.snapshot_id') DIFF_FROM_REMOTE_CMD="--parent ${DIFF_FROM_REMOTE}" fi -restic backup $DIFF_FROM_REMOTE_CMD --tag "${BACKUP_NAME}" $LOCAL_PATHS -restic forget --keep-last ${RESTIC_KEEP_LAST} --prune \ No newline at end of file +restic backup --insecure-tls $DIFF_FROM_REMOTE_CMD --tag "${BACKUP_NAME}" $LOCAL_PATHS +restic forget --insecure-tls --keep-last ${RESTIC_KEEP_LAST} --prune \ No newline at end of file diff --git a/test/integration/rsync/list.sh b/test/integration/rsync/list.sh index 5ba42433..26dca677 100755 --- a/test/integration/rsync/list.sh +++ b/test/integration/rsync/list.sh @@ -3,8 +3,8 @@ set +x set -euo pipefail CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" source "${CUR_DIR}/init.sh" -ssh -i "${BACKUP_SSH_KEY}" -o "StrictHostKeyChecking no" "${BACKUP_REMOTE_SERVER}" ls -d -1 "${BACKUP_REMOTE_DIR}/*" | while IFS= read -r backup_name ; do +ssh -i "${BACKUP_SSH_KEY}" -o "StrictHostKeyChecking no" "${BACKUP_REMOTE_SERVER}" ls -d -1 "${BACKUP_REMOTE_DIR}/*" 2>/dev/null | while IFS= read -r backup_name ; do backup_name=${backup_name#"$BACKUP_REMOTE_DIR"} - ssh -i "${BACKUP_SSH_KEY}" -o "StrictHostKeyChecking no" "${BACKUP_REMOTE_SERVER}" cat "${BACKUP_REMOTE_DIR}/${backup_name}/default/metadata.json" | jq -c -r -M '.' + ssh -i "${BACKUP_SSH_KEY}" -o "StrictHostKeyChecking no" "${BACKUP_REMOTE_SERVER}" cat "${BACKUP_REMOTE_DIR}/${backup_name}/default/metadata.json" 2>/dev/null | jq -c -r -M '. + {upload_date: .creation_date}' done -set -x \ No newline at end of file +set -x diff --git a/test/integration/run.sh b/test/integration/run.sh index 36d3ab68..450d041c 100755 --- a/test/integration/run.sh +++ b/test/integration/run.sh @@ -2,13 +2,13 @@ set -x set -e -CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +export CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" mkdir -p "${CUR_DIR}/_coverage_/" rm -rf "${CUR_DIR}/_coverage_/*" source "${CUR_DIR}/.env" -export CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION:-23.3} +export CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION:-24.3} if [[ "${CLICKHOUSE_VERSION}" =~ ^2[1-9]+ || "${CLICKHOUSE_VERSION}" == "head" ]]; then export CLICKHOUSE_IMAGE=${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server} else @@ -16,6 +16,7 @@ else fi export CLICKHOUSE_BACKUP_BIN="$(pwd)/clickhouse-backup/clickhouse-backup-race" export LOG_LEVEL=${LOG_LEVEL:-info} +export TEST_LOG_LEVEL=${TEST_LOG_LEVEL:-info} if [[ -f "${CUR_DIR}/credentials.json" ]]; then export GCS_TESTS=${GCS_TESTS:-1} @@ -27,11 +28,12 @@ export GLACIER_TESTS=${GLACIER_TESTS:-0} export AZURE_TESTS=${AZURE_TESTS:-1} export RUN_ADVANCED_TESTS=${RUN_ADVANCED_TESTS:-1} +export GODEBUG=${GODEBUG:-} export S3_DEBUG=${S3_DEBUG:-false} export GCS_DEBUG=${GCS_DEBUG:-false} export FTP_DEBUG=${FTP_DEBUG:-false} export SFTP_DEBUG=${SFTP_DEBUG:-false} -export GODEBUG=${GODEBUG:-} +export AZBLOB_DEBUG=${AZBLOB_DEBUG:-false} export CLICKHOUSE_DEBUG=${CLICKHOUSE_DEBUG:-false} if [[ "${CLICKHOUSE_VERSION}" == 2* || "${CLICKHOUSE_VERSION}" == "head" ]]; then @@ -40,11 +42,66 @@ else export COMPOSE_FILE=docker-compose.yml fi -docker-compose -f ${CUR_DIR}/${COMPOSE_FILE} down --remove-orphans + +pids=() +for project in $(docker compose -f ${CUR_DIR}/${COMPOSE_FILE} ls --all -q); do + docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project} --progress plain down --remove-orphans --volumes --timeout=1 & + pids+=($!) +done + +for pid in "${pids[@]}"; do + if wait "$pid"; then + echo "$pid docker compose down successful" + else + echo "$pid docker compose down failed. Exiting." + exit 1 # Exit with an error code if any command fails + fi +done + docker volume prune -f make clean build-race-docker build-race-fips-docker -docker-compose -f ${CUR_DIR}/${COMPOSE_FILE} up -d -docker-compose -f ${CUR_DIR}/${COMPOSE_FILE} exec minio mc alias list -go test -parallel ${RUN_PARALLEL:-$(nproc)} -timeout ${TESTS_TIMEOUT:-30m} -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v ${CUR_DIR}/integration_test.go -go tool covdata textfmt -i "${CUR_DIR}/_coverage_/" -o "${CUR_DIR}/_coverage_/coverage.out" \ No newline at end of file +export RUN_PARALLEL=${RUN_PARALLEL:-1} + +docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --progress=quiet pull + +pids=() +for ((i = 0; i < RUN_PARALLEL; i++)); do + docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name project${i} --progress plain up -d & + pids+=($!) +done + +for pid in "${pids[@]}"; do + if wait "$pid"; then + echo "$pid docker compose up successful" + else + echo "$pid docker compose up failed. Exiting." + exit 1 # Exit with an error code if any command fails + fi +done + +set +e +go test -parallel ${RUN_PARALLEL} -race -timeout ${TEST_TIMEOUT:-60m} -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v ${CUR_DIR}/integration_test.go +TEST_FAILED=$? +set -e + +if [[ "0" == "${TEST_FAILED}" ]]; then + go tool covdata textfmt -i "${CUR_DIR}/_coverage_/" -o "${CUR_DIR}/_coverage_/coverage.out" +fi + +if [[ "1" == "${CLEAN_AFTER:-0}" || "0" == "${TEST_FAILED}" ]]; then + pids=() + for project in $(docker compose -f ${CUR_DIR}/${COMPOSE_FILE} ls --all -q); do + docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project} --progress plain down --remove-orphans --volumes --timeout=1 & + pids+=($!) + done + + for pid in "${pids[@]}"; do + if wait "$pid"; then + echo "$pid docker compose down successful" + else + echo "$pid docker compose down failed. Exiting." + exit 1 # Exit with an error code if any command fails + fi + done +fi \ No newline at end of file diff --git a/test/testflows/README.md b/test/testflows/README.md index 2b06f028..9e83cdd4 100644 --- a/test/testflows/README.md +++ b/test/testflows/README.md @@ -9,7 +9,7 @@ To execute tests, you will need: * Python 3.8 or higher (`python3`) * The latest version of Go (`https://go.dev/doc/install`) * TestFlows Python library (`https://testflows.com/`) -* `docker` and `docker-compose` +* `docker` and `docker compose` To install all necessary Python packages, execute: ```bash @@ -17,15 +17,15 @@ pip3 install -r ./test/testflows/requirements.txt ``` ## Execution -Make clickhouse-backup binary +Make the clickhouse-backup binary: ```bash make build-race-docker ``` -Some environment variables required to be set up before test execution: +Some environment variables must be set up before test execution: * `export CLICKHOUSE_TESTS_DIR=/home/username/clickhouse-backup/test/testflows/clickhouse_backup` - (this variable must point to the folder containing `regression.py`) -* In order to test cloud platforms (AWS S3 and GCS), you will need the following variables to contain valid credentials (otherwise, the corresponding tests will fail): +* In order to test cloud platforms (AWS S3 and GCS), you will need the following variables to contain valid credentials, otherwise the corresponding tests will fail: - `QA_AWS_ACCESS_KEY` - `QA_AWS_ENDPOINT` - `QA_AWS_SECRET_KEY` @@ -34,7 +34,7 @@ Some environment variables required to be set up before test execution: - `QA_GCS_CRED_JSON` - `QA_GCS_CRED_JSON_ENCODED` -You can do it with something like that: +Sample setup script: ```bash cat > /home/username/clickhouse-backup/test/testflows/.env <&1 | tee' cluster.command(None, command) diff --git a/test/testflows/clickhouse_backup/tests/snapshots/cli.py.cli.snapshot b/test/testflows/clickhouse_backup/tests/snapshots/cli.py.cli.snapshot index d932e1c7..4a2ea3ed 100644 --- a/test/testflows/clickhouse_backup/tests/snapshots/cli.py.cli.snapshot +++ b/test/testflows/clickhouse_backup/tests/snapshots/cli.py.cli.snapshot @@ -1,6 +1,6 @@ -default_config = r"""'[\'general:\', \' remote_storage: none\', \' disable_progress_bar: true\', \' backups_to_keep_local: 0\', \' backups_to_keep_remote: 0\', \' log_level: info\', \' allow_empty_backups: false\', \' use_resumable_state: true\', \' restore_schema_on_cluster: ""\', \' upload_by_part: true\', \' download_by_part: true\', \' restore_database_mapping: {}\', \' retries_on_failure: 3\', \' retries_pause: 30s\', \' watch_interval: 1h\', \' full_interval: 24h\', \' watch_backup_name_template: shard{shard}-{type}-{time:20060102150405}\', \' sharded_operation_mode: ""\', \' retriesduration: 100ms\', \' watchduration: 1h0m0s\', \' fullduration: 24h0m0s\', \'clickhouse:\', \' username: default\', \' password: ""\', \' host: localhost\', \' port: 9000\', \' disk_mapping: {}\', \' skip_tables:\', \' - system.*\', \' - INFORMATION_SCHEMA.*\', \' - information_schema.*\', \' - _temporary_and_external_tables.*\', \' skip_table_engines: []\', \' timeout: 5m\', \' freeze_by_part: false\', \' freeze_by_part_where: ""\', \' use_embedded_backup_restore: false\', \' embedded_backup_disk: ""\', \' backup_mutations: true\', \' restore_as_attach: false\', \' check_parts_columns: true\', \' secure: false\', \' skip_verify: false\', \' sync_replicated_tables: false\', \' log_sql_queries: true\', \' config_dir: /etc/clickhouse-server/\', \' restart_command: exec:systemctl restart clickhouse-server\', \' ignore_not_exists_error_during_freeze: true\', \' check_replicas_before_attach: true\', \' tls_key: ""\', \' tls_cert: ""\', \' tls_ca: ""\', \' debug: false\', \'s3:\', \' access_key: ""\', \' secret_key: ""\', \' bucket: ""\', \' endpoint: ""\', \' region: us-east-1\', \' acl: private\', \' assume_role_arn: ""\', \' force_path_style: false\', \' path: ""\', \' object_disk_path: ""\', \' disable_ssl: false\', \' compression_level: 1\', \' compression_format: tar\', \' sse: ""\', \' sse_kms_key_id: ""\', \' sse_customer_algorithm: ""\', \' sse_customer_key: ""\', \' sse_customer_key_md5: ""\', \' sse_kms_encryption_context: ""\', \' disable_cert_verification: false\', \' use_custom_storage_class: false\', \' storage_class: STANDARD\', \' custom_storage_class_map: {}\', \' part_size: 0\', \' allow_multipart_download: false\', \' object_labels: {}\', \' debug: false\', \'gcs:\', \' credentials_file: ""\', \' credentials_json: ""\', \' credentials_json_encoded: ""\', \' bucket: ""\', \' path: ""\', \' object_disk_path: ""\', \' compression_level: 1\', \' compression_format: tar\', \' debug: false\', \' endpoint: ""\', \' storage_class: STANDARD\', \' object_labels: {}\', \' custom_storage_class_map: {}\', \'cos:\', \' url: ""\', \' timeout: 2m\', \' secret_id: ""\', \' secret_key: ""\', \' path: ""\', \' compression_format: tar\', \' compression_level: 1\', \' debug: false\', \'api:\', \' listen: localhost:7171\', \' enable_metrics: true\', \' enable_pprof: false\', \' username: ""\', \' password: ""\', \' secure: false\', \' certificate_file: ""\', \' private_key_file: ""\', \' ca_cert_file: ""\', \' ca_key_file: ""\', \' create_integration_tables: false\', \' integration_tables_host: ""\', \' allow_parallel: false\', \' complete_resumable_after_restart: true\', \'ftp:\', \' address: ""\', \' timeout: 2m\', \' username: ""\', \' password: ""\', \' tls: false\', \' path: ""\', \' object_disk_path: ""\', \' compression_format: tar\', \' compression_level: 1\', \' debug: false\', \'sftp:\', \' address: ""\', \' port: 22\', \' username: ""\', \' password: ""\', \' key: ""\', \' path: ""\', \' object_disk_path: ""\', \' compression_format: tar\', \' compression_level: 1\', \' debug: false\', \'azblob:\', \' endpoint_schema: https\', \' endpoint_suffix: core.windows.net\', \' account_name: ""\', \' account_key: ""\', \' sas: ""\', \' use_managed_identity: false\', \' container: ""\', \' path: ""\', \' object_disk_path: ""\', \' compression_level: 1\', \' compression_format: tar\', \' sse_key: ""\', \' buffer_size: 0\', \' buffer_count: 3\', \' timeout: 15m\', \'custom:\', \' upload_command: ""\', \' download_command: ""\', \' list_command: ""\', \' delete_command: ""\', \' command_timeout: 4h\', \' commandtimeoutduration: 4h0m0s\']'""" +default_config = r"""'[\'general:\', \' remote_storage: none\', \' backups_to_keep_local: 0\', \' backups_to_keep_remote: 0\', \' log_level: info\', \' allow_empty_backups: false\', \' use_resumable_state: true\', \' restore_schema_on_cluster: ""\', \' upload_by_part: true\', \' download_by_part: true\', \' restore_database_mapping: {}\', \' restore_table_mapping: {}\', \' retries_on_failure: 3\', \' retries_pause: 30s\', \' watch_interval: 1h\', \' full_interval: 24h\', \' watch_backup_name_template: shard{shard}-{type}-{time:20060102150405}\', \' sharded_operation_mode: ""\', \' cpu_nice_priority: 15\', \' io_nice_priority: idle\', \' rbac_backup_always: true\', \' rbac_conflict_resolution: recreate\', \' retriesduration: 100ms\', \' watchduration: 1h0m0s\', \' fullduration: 24h0m0s\', \'clickhouse:\', \' username: default\', \' password: ""\', \' host: localhost\', \' port: 9000\', \' disk_mapping: {}\', \' skip_tables:\', \' - system.*\', \' - INFORMATION_SCHEMA.*\', \' - information_schema.*\', \' - _temporary_and_external_tables.*\', \' skip_table_engines: []\', \' timeout: 30m\', \' freeze_by_part: false\', \' freeze_by_part_where: ""\', \' use_embedded_backup_restore: false\', \' embedded_backup_disk: ""\', \' backup_mutations: true\', \' restore_as_attach: false\', \' check_parts_columns: true\', \' secure: false\', \' skip_verify: false\', \' sync_replicated_tables: false\', \' log_sql_queries: true\', \' config_dir: /etc/clickhouse-server/\', \' restart_command: exec:systemctl restart clickhouse-server\', \' ignore_not_exists_error_during_freeze: true\', \' check_replicas_before_attach: true\', \' tls_key: ""\', \' tls_cert: ""\', \' tls_ca: ""\', \' debug: false\', \'s3:\', \' access_key: ""\', \' secret_key: ""\', \' bucket: ""\', \' endpoint: ""\', \' region: us-east-1\', \' acl: private\', \' assume_role_arn: ""\', \' force_path_style: false\', \' path: ""\', \' object_disk_path: ""\', \' disable_ssl: false\', \' compression_level: 1\', \' compression_format: tar\', \' sse: ""\', \' sse_kms_key_id: ""\', \' sse_customer_algorithm: ""\', \' sse_customer_key: ""\', \' sse_customer_key_md5: ""\', \' sse_kms_encryption_context: ""\', \' disable_cert_verification: false\', \' use_custom_storage_class: false\', \' storage_class: STANDARD\', \' custom_storage_class_map: {}\', \' part_size: 0\', \' allow_multipart_download: false\', \' object_labels: {}\', \' request_payer: ""\', \' check_sum_algorithm: ""\', \' debug: false\', \'gcs:\', \' credentials_file: ""\', \' credentials_json: ""\', \' credentials_json_encoded: ""\', \' embedded_access_key: ""\', \' embedded_secret_key: ""\', \' skip_credentials: false\', \' bucket: ""\', \' path: ""\', \' object_disk_path: ""\', \' compression_level: 1\', \' compression_format: tar\', \' debug: false\', \' force_http: false\', \' endpoint: ""\', \' storage_class: STANDARD\', \' object_labels: {}\', \' custom_storage_class_map: {}\', \' chunk_size: 0\', \'cos:\', \' url: ""\', \' timeout: 2m\', \' secret_id: ""\', \' secret_key: ""\', \' path: ""\', \' compression_format: tar\', \' compression_level: 1\', \' debug: false\', \'api:\', \' listen: localhost:7171\', \' enable_metrics: true\', \' enable_pprof: false\', \' username: ""\', \' password: ""\', \' secure: false\', \' certificate_file: ""\', \' private_key_file: ""\', \' ca_cert_file: ""\', \' ca_key_file: ""\', \' create_integration_tables: false\', \' integration_tables_host: ""\', \' allow_parallel: false\', \' complete_resumable_after_restart: true\', \' watch_is_main_process: false\', \'ftp:\', \' address: ""\', \' timeout: 2m\', \' username: ""\', \' password: ""\', \' tls: false\', \' skip_tls_verify: false\', \' path: ""\', \' object_disk_path: ""\', \' compression_format: tar\', \' compression_level: 1\', \' debug: false\', \'sftp:\', \' address: ""\', \' port: 22\', \' username: ""\', \' password: ""\', \' key: ""\', \' path: ""\', \' object_disk_path: ""\', \' compression_format: tar\', \' compression_level: 1\', \' debug: false\', \'azblob:\', \' endpoint_schema: https\', \' endpoint_suffix: core.windows.net\', \' account_name: ""\', \' account_key: ""\', \' sas: ""\', \' use_managed_identity: false\', \' container: ""\', \' path: ""\', \' object_disk_path: ""\', \' compression_level: 1\', \' compression_format: tar\', \' sse_key: ""\', \' buffer_size: 0\', \' buffer_count: 3\', \' timeout: 4h\', \' debug: false\', \'custom:\', \' upload_command: ""\', \' download_command: ""\', \' list_command: ""\', \' delete_command: ""\', \' command_timeout: 4h\', \' commandtimeoutduration: 4h0m0s\']'""" -help_flag = r"""'NAME:\n clickhouse-backup - Tool for easy backup of ClickHouse with cloud supportUSAGE:\n clickhouse-backup [-t, --tables=.] DESCRIPTION:\n Run as \'root\' or \'clickhouse\' userCOMMANDS:\n tables List of tables, exclude skip_tables\n create Create new backup\n create_remote Create and upload new backup\n upload Upload backup to remote storage\n list List of backups\n download Download backup from remote storage\n restore Create schema and restore data from backup\n restore_remote Download and restore\n delete Delete specific backup\n default-config Print default config\n print-config Print current config merged with environment variables\n clean Remove data in \'shadow\' folder from all \'path\' folders available from \'system.disks\'\n clean_remote_broken Remove all broken remote backups\n watch Run infinite loop which create full + incremental backup sequence to allow efficient backup sequences\n server Run API server\n help, h Shows a list of commands or help for one commandGLOBAL OPTIONS:\n --config value, -c value Config \'FILE\' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]\n --help, -h show help\n --version, -v print the version'""" +help_flag = r"""'NAME:\n clickhouse-backup - Tool for easy backup of ClickHouse with cloud supportUSAGE:\n clickhouse-backup [-t, --tables=.
] DESCRIPTION:\n Run as \'root\' or \'clickhouse\' userCOMMANDS:\n tables List of tables, exclude skip_tables\n create Create new backup\n create_remote Create and upload new backup\n upload Upload backup to remote storage\n list List of backups\n download Download backup from remote storage\n restore Create schema and restore data from backup\n restore_remote Download and restore\n delete Delete specific backup\n default-config Print default config\n print-config Print current config merged with environment variables\n clean Remove data in \'shadow\' folder from all \'path\' folders available from \'system.disks\'\n clean_remote_broken Remove all broken remote backups\n watch Run infinite loop which create full + incremental backup sequence to allow efficient backup sequences\n server Run API server\n help, h Shows a list of commands or help for one commandGLOBAL OPTIONS:\n --config value, -c value Config \'FILE\' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]\n --environment-override value, --env value override any environment variable via CLI parameter\n --help, -h show help\n --version, -v print the version'""" -cli_usage = r"""'NAME:\n clickhouse-backup - Tool for easy backup of ClickHouse with cloud supportUSAGE:\n clickhouse-backup [-t, --tables=.
] DESCRIPTION:\n Run as \'root\' or \'clickhouse\' userCOMMANDS:\n tables List of tables, exclude skip_tables\n create Create new backup\n create_remote Create and upload new backup\n upload Upload backup to remote storage\n list List of backups\n download Download backup from remote storage\n restore Create schema and restore data from backup\n restore_remote Download and restore\n delete Delete specific backup\n default-config Print default config\n print-config Print current config merged with environment variables\n clean Remove data in \'shadow\' folder from all \'path\' folders available from \'system.disks\'\n clean_remote_broken Remove all broken remote backups\n watch Run infinite loop which create full + incremental backup sequence to allow efficient backup sequences\n server Run API server\n help, h Shows a list of commands or help for one commandGLOBAL OPTIONS:\n --config value, -c value Config \'FILE\' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]\n --help, -h show help\n --version, -v print the version'""" +cli_usage = r"""'NAME:\n clickhouse-backup - Tool for easy backup of ClickHouse with cloud supportUSAGE:\n clickhouse-backup [-t, --tables=.
] DESCRIPTION:\n Run as \'root\' or \'clickhouse\' userCOMMANDS:\n tables List of tables, exclude skip_tables\n create Create new backup\n create_remote Create and upload new backup\n upload Upload backup to remote storage\n list List of backups\n download Download backup from remote storage\n restore Create schema and restore data from backup\n restore_remote Download and restore\n delete Delete specific backup\n default-config Print default config\n print-config Print current config merged with environment variables\n clean Remove data in \'shadow\' folder from all \'path\' folders available from \'system.disks\'\n clean_remote_broken Remove all broken remote backups\n watch Run infinite loop which create full + incremental backup sequence to allow efficient backup sequences\n server Run API server\n help, h Shows a list of commands or help for one commandGLOBAL OPTIONS:\n --config value, -c value Config \'FILE\' name. (default: "/etc/clickhouse-backup/config.yml") [$CLICKHOUSE_BACKUP_CONFIG]\n --environment-override value, --env value override any environment variable via CLI parameter\n --help, -h show help\n --version, -v print the version'""" diff --git a/test/testflows/clickhouse_backup/tests/steps.py b/test/testflows/clickhouse_backup/tests/steps.py index 94695c36..f6cea81d 100644 --- a/test/testflows/clickhouse_backup/tests/steps.py +++ b/test/testflows/clickhouse_backup/tests/steps.py @@ -117,8 +117,8 @@ def populate_table(self, node, table_name, columns, database="default", size=10, str_random_schema = ", ".join(random_schema) str_insert_columns = ", ".join(insert_columns) - - node.query(f"INSERT INTO {database}.{table_name} ({str_insert_columns}) SELECT * FROM generateRandom('{str_random_schema}', NULL, 10, 2) LIMIT {size}") + # @todo return NULL to random_seed generateRandom parameter when resolve https://github.com/ClickHouse/ClickHouse/issues/62092 + node.query(f"INSERT INTO {database}.{table_name} ({str_insert_columns}) SELECT * FROM generateRandom('{str_random_schema}', {random.randint(1,10000)}, 10, 2) LIMIT {size}") @TestStep(Given) diff --git a/test/testflows/clickhouse_backup/tests/views.py b/test/testflows/clickhouse_backup/tests/views.py index 2cc97eb9..888f1194 100644 --- a/test/testflows/clickhouse_backup/tests/views.py +++ b/test/testflows/clickhouse_backup/tests/views.py @@ -97,7 +97,11 @@ def window_view(self): """Test that window view is handled properly by clickhouse-backup. """ base_table_name = self.context.views_base_name - if os.environ.get('CLICKHOUSE_VERSION', '22.8') >= '22.6': + if os.environ.get('CLICKHOUSE_VERSION', '24.3') >= '24.3': + create_query = f"CREATE WINDOW VIEW {base_table_name}_wview " \ + f"ENGINE AggregatingMergeTree() ORDER BY t AS SELECT count(Version) v, tumbleStart(w_id) t " \ + f"FROM default.{base_table_name} GROUP BY tumble(Time, INTERVAL '10' SECOND) AS w_id SETTINGS allow_experimental_analyzer=0" + elif os.environ.get('CLICKHOUSE_VERSION', '24.3') >= '22.6': create_query = f"CREATE WINDOW VIEW {base_table_name}_wview " \ f"ENGINE AggregatingMergeTree() ORDER BY t AS SELECT count(Version) v, tumbleStart(w_id) t " \ f"FROM default.{base_table_name} GROUP BY tumble(Time, INTERVAL '10' SECOND) AS w_id" diff --git a/test/testflows/helpers/argparser.py b/test/testflows/helpers/argparser.py index b8bd1124..a272917b 100644 --- a/test/testflows/helpers/argparser.py +++ b/test/testflows/helpers/argparser.py @@ -3,4 +3,4 @@ def argparser(parser): """ parser.add_argument("--local", action="store_true", - help="run regression in local mode without docker-compose down", default=True) + help="run regression in local mode without docker compose down", default=True) diff --git a/test/testflows/helpers/cluster.py b/test/testflows/helpers/cluster.py index e33ddf85..2f02bcfb 100644 --- a/test/testflows/helpers/cluster.py +++ b/test/testflows/helpers/cluster.py @@ -2,10 +2,9 @@ import inspect import os import tempfile +import testflows.settings as settings import threading import time - -import testflows.settings as settings from testflows.asserts import error from testflows.connect import Shell as ShellBase from testflows.core import * @@ -35,7 +34,7 @@ def __exit__(self, shell_type, shell_value, traceback): # to terminate any open shell commands. # This is needed for example # to solve a problem with - # 'docker-compose exec {name} bash --noediting' + # 'docker compose exec {name} bash --noediting' # that does not clean up open bash processes # if not exited normally for i in range(10): @@ -408,13 +407,13 @@ def query(self, sql, expect_message=None, exitcode=None, steps=True, no_checks=F class Cluster(object): - """Simple object around docker-compose cluster. + """Simple object around docker compose cluster. """ def __init__(self, local=False, configs_dir=None, nodes=None, - docker_compose="docker-compose", docker_compose_project_dir=None, + docker_compose="docker compose", docker_compose_project_dir=None, docker_compose_file="docker-compose.yml", environ=None): @@ -448,7 +447,7 @@ def __init__(self, local=False, if not os.path.exists(docker_compose_file_path): raise TypeError(f"docker compose file '{docker_compose_file_path}' does not exist") - self.docker_compose += f" --ansi never --project-directory \"{docker_compose_project_dir}\" --file \"{docker_compose_file_path}\"" + self.docker_compose += f" --ansi never --progress plain --project-directory \"{docker_compose_project_dir}\" --file \"{docker_compose_file_path}\"" self.lock = threading.Lock() @property @@ -508,7 +507,7 @@ def node_wait_healthy(self, node, timeout=300): while True: try: c = self.control_shell(f"{self.docker_compose} ps {node} | grep {node}", timeout=timeout) - if c.exitcode == 0 and 'Up (healthy)' in c.output: + if c.exitcode == 0 and '(healthy)' in c.output: return except IOError: raise @@ -612,7 +611,7 @@ def close_bash(self, node): del self.shells[shell_id] def __enter__(self): - with Given("docker-compose cluster"): + with Given("docker compose cluster"): self.up() return self @@ -634,7 +633,7 @@ def node(self, node_name): return Node(self, node_name) def down(self, timeout=300): - """Bring cluster down by executing docker-compose down.""" + """Bring cluster down by executing docker compose down.""" # add message to each clickhouse-server.log if settings.debug: @@ -691,7 +690,7 @@ def up(self, timeout=120): with And("I list environment variables to show their values"): self.command(None, "env | grep CLICKHOUSE") - with Given("docker-compose"): + with Given("docker compose"): max_attempts = 5 max_up_attempts = 1 @@ -700,9 +699,9 @@ def up(self, timeout=120): with By("checking if any containers are already running"): self.command(None, f"{self.docker_compose} ps | tee") - with And("executing docker-compose down just in case it is up"): + with And("executing docker compose down just in case it is up"): cmd = self.command( - None, f"{self.docker_compose} down --timeout=10 2>&1 | tee", exitcode=None, timeout=timeout + None, f"{self.docker_compose} down --timeout=1 2>&1 | tee", exitcode=None, timeout=timeout ) if cmd.exitcode != 0: continue @@ -710,7 +709,7 @@ def up(self, timeout=120): with And("checking if any containers are still left running"): self.command(None, f"{self.docker_compose} ps | tee") - with And("executing docker-compose up"): + with And("executing docker compose up"): for up_attempt in range(max_up_attempts): with By(f"attempt {up_attempt}/{max_up_attempts}"): cmd = self.command( @@ -729,7 +728,7 @@ def up(self, timeout=120): break if cmd.exitcode != 0 or "is unhealthy" in cmd.output or "Exit" in ps_cmd.output: - fail("could not bring up docker-compose cluster") + fail("could not bring up docker compose cluster") with Then("wait all nodes report healthy"): for node_name in self.nodes["clickhouse"]: diff --git a/test/testflows/requirements.txt b/test/testflows/requirements.txt index 541c9085..aab89fbe 100644 --- a/test/testflows/requirements.txt +++ b/test/testflows/requirements.txt @@ -1,4 +1,4 @@ -testflows==1.9.71 +testflows==2.4.11 requests setuptools PyYAML \ No newline at end of file diff --git a/test/testflows/run.sh b/test/testflows/run.sh old mode 100644 new mode 100755 index 92eb3706..e37b7ac9 --- a/test/testflows/run.sh +++ b/test/testflows/run.sh @@ -2,7 +2,7 @@ CUR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" mkdir -p "${CUR_DIR}/_coverage_/" rm -rf "${CUR_DIR}/_coverage_/*" source "${CUR_DIR}/.env" -export CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION:-23.3} +export CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION:-24.3} if [[ "${CLICKHOUSE_VERSION}" =~ ^2[1-9]+ || "${CLICKHOUSE_VERSION}" == "head" ]]; then export CLICKHOUSE_IMAGE=${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server} else @@ -10,4 +10,4 @@ else fi make clean build-race-docker python3 "${CUR_DIR}/clickhouse_backup/regression.py" --debug --only="${RUN_TESTS:-*}" -go tool covdata textfmt -i "${CUR_DIR}/_coverage_/" -o "${CUR_DIR}/_coverage_/coverage.out" \ No newline at end of file +go tool covdata textfmt -i "${CUR_DIR}/_coverage_/" -o "${CUR_DIR}/_coverage_/coverage.out"