From d510c4c01c57bc789eaabb561e83a09c375707dc Mon Sep 17 00:00:00 2001 From: Slach Date: Wed, 13 Dec 2023 07:42:27 +0400 Subject: [PATCH] fix object_disk.CopyObject during restore to allow use properly S3 endpoint, remove fips version from build-race in Makefile --- ChangeLog.md | 4 ++++ Makefile | 9 +++++---- pkg/backup/restore.go | 2 +- pkg/storage/object_disk/object_disk.go | 2 +- pkg/storage/s3.go | 2 +- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 93756f11..53eeefb6 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,7 @@ +# v2.4.13 +BUG FIXES +- fix object_disk.CopyObject during restore to allow use properly S3 endpoint + # v2.4.12 BUG FIXES - fix CopyObject to use simple CopyObject call, instead of multipart for zero object size, for backup S3 disks diff --git a/Makefile b/Makefile index 15368c36..263f96a5 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,8 @@ define DESC = Support of incremental backups on remote storages' endef GO_BUILD = go build -buildvcs=false -ldflags "-X 'main.version=$(VERSION)' -X 'main.gitCommit=$(GIT_COMMIT)' -X 'main.buildDate=$(DATE)'" -GO_BUILD_STATIC = go build -buildvcs=false -ldflags "-X 'main.version=$(VERSION)-fips' -X 'main.gitCommit=$(GIT_COMMIT)' -X 'main.buildDate=$(DATE)' -linkmode=external -extldflags '-static'" +GO_BUILD_STATIC = go build -buildvcs=false -ldflags "-X 'main.version=$(VERSION)' -X 'main.gitCommit=$(GIT_COMMIT)' -X 'main.buildDate=$(DATE)' -linkmode=external -extldflags '-static'" +GO_BUILD_STATIC_FIPS = go build -buildvcs=false -ldflags "-X 'main.version=$(VERSION)-fips' -X 'main.gitCommit=$(GIT_COMMIT)' -X 'main.buildDate=$(DATE)' -linkmode=external -extldflags '-static'" PKG_FILES = build/$(NAME)_$(VERSION).amd64.deb build/$(NAME)_$(VERSION).arm64.deb build/$(NAME)-$(VERSION)-1.amd64.rpm build/$(NAME)-$(VERSION)-1.arm64.rpm HOST_OS = $(shell bash -c 'source <(go env) && echo $$GOHOSTOS') HOST_ARCH = $(shell bash -c 'source <(go env) && echo $$GOHOSTARCH') @@ -55,7 +56,7 @@ build/linux/arm64/$(NAME)-fips build/darwin/arm64/$(NAME)-fips: GOARCH = arm64 build/linux/amd64/$(NAME)-fips build/linux/arm64/$(NAME)-fips: GOOS = linux build/darwin/amd64/$(NAME)-fips build/darwin/arm64/$(NAME)-fips: GOOS = darwin build/linux/amd64/$(NAME)-fips build/darwin/amd64/$(NAME)-fips: - CC=musl-gcc GOEXPERIMENT=boringcrypto CGO_ENABLED=1 GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO_BUILD_STATIC) -o $@ ./cmd/$(NAME) && \ + CC=musl-gcc GOEXPERIMENT=boringcrypto CGO_ENABLED=1 GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO_BUILD_STATIC_FIPS) -o $@ ./cmd/$(NAME) && \ go tool nm $@ > /tmp/$(NAME)-fips-tags.txt && \ grep '_Cfunc__goboringcrypto_' /tmp/$(NAME)-fips-tags.txt 1> /dev/null && \ rm -fv /tmp/$(NAME)-fips-tags.txt @@ -63,7 +64,7 @@ build/linux/amd64/$(NAME)-fips build/darwin/amd64/$(NAME)-fips: # TODO remove ugly workaround, https://www.perplexity.ai/search/2ead4c04-060a-4d78-a75f-f26835238438 build/linux/arm64/$(NAME)-fips build/darwin/arm64/$(NAME)-fips: bash -xce 'if [[ ! -f ~/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc ]]; then wget -q -P ~ https://musl.cc/aarch64-linux-musl-cross.tgz; tar -xvf ~/aarch64-linux-musl-cross.tgz -C ~; fi' && \ - CC=~/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc GOEXPERIMENT=boringcrypto CGO_ENABLED=1 GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO_BUILD_STATIC) -o $@ ./cmd/$(NAME) && \ + CC=~/aarch64-linux-musl-cross/bin/aarch64-linux-musl-gcc GOEXPERIMENT=boringcrypto CGO_ENABLED=1 GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO_BUILD_STATIC_FIPS) -o $@ ./cmd/$(NAME) && \ go tool nm $@ > /tmp/$(NAME)-fips-tags.txt && \ grep '_Cfunc__goboringcrypto_' /tmp/$(NAME)-fips-tags.txt 1> /dev/null && \ rm -fv /tmp/$(NAME)-fips-tags.txt @@ -128,7 +129,7 @@ $(NAME)/$(NAME)-race: build-race-fips: $(NAME)/$(NAME)-race-fips $(NAME)/$(NAME)-race-fips: - CC=musl-gcc GOEXPERIMENT=boringcrypto CGO_ENABLED=1 $(GO_BUILD_STATIC) -cover -gcflags "all=-N -l" -race -o $@ ./cmd/$(NAME) + CC=musl-gcc GOEXPERIMENT=boringcrypto CGO_ENABLED=1 $(GO_BUILD_STATIC_FIPS) -cover -gcflags "all=-N -l" -race -o $@ ./cmd/$(NAME) # run `docker buildx create --use` first time diff --git a/pkg/backup/restore.go b/pkg/backup/restore.go index 98302f08..a104c0b0 100644 --- a/pkg/backup/restore.go +++ b/pkg/backup/restore.go @@ -868,7 +868,7 @@ func (b *Backuper) downloadObjectDiskParts(ctx context.Context, backupName strin return err } if objMeta.StorageObjectCount < 1 && objMeta.Version != object_disk.VersionRelativePath { - return fmt.Errorf("%s: invalid object_dist.Metadata: %#v", fPath, objMeta) + return fmt.Errorf("%s: invalid object_disk.Metadata: %#v", fPath, objMeta) } var srcBucket, srcKey string for _, storageObject := range objMeta.StorageObjects { diff --git a/pkg/storage/object_disk/object_disk.go b/pkg/storage/object_disk/object_disk.go index 7af36d74..f0b8d178 100644 --- a/pkg/storage/object_disk/object_disk.go +++ b/pkg/storage/object_disk/object_disk.go @@ -435,7 +435,6 @@ func makeObjectDiskConnection(ctx context.Context, ch *clickhouse.ClickHouse, cf if err != nil { return nil, err } - s3cfg.Endpoint = s3URL.Scheme + "://" + s3URL.Host if cfg.S3.Concurrency > 0 { s3cfg.Concurrency = cfg.S3.Concurrency } @@ -471,6 +470,7 @@ func makeObjectDiskConnection(ctx context.Context, ch *clickhouse.ClickHouse, cf s3cfg.Path = strings.Trim(s3URL.Path, "/") s3cfg.ForcePathStyle = false } else { + s3cfg.Endpoint = s3URL.Scheme + "://" + s3URL.Host pathItems := strings.Split(strings.Trim(s3URL.Path, "/"), "/") s3cfg.Bucket = pathItems[0] s3cfg.Path = path.Join(pathItems[1:]...) diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go index 22e3161b..e217f92b 100644 --- a/pkg/storage/s3.go +++ b/pkg/storage/s3.go @@ -452,8 +452,8 @@ func (s *S3) remotePager(ctx context.Context, s3Path string, recursive bool, pro } func (s *S3) CopyObject(ctx context.Context, srcBucket, srcKey, dstKey string) (int64, error) { - s.Log.Debugf("S3->CopyObject %s/%s -> %s/%s", srcBucket, srcKey, s.Config.Bucket, dstKey) dstKey = path.Join(s.Config.ObjectDiskPath, dstKey) + s.Log.Debugf("S3->CopyObject %s/%s -> %s/%s", srcBucket, srcKey, s.Config.Bucket, dstKey) if strings.Contains(s.Config.Endpoint, "storage.googleapis.com") { params := &s3.CopyObjectInput{ Bucket: aws.String(s.Config.Bucket),