From dfa2811f186c3abc1434d3a6a93851fd8b6c1950 Mon Sep 17 00:00:00 2001 From: Slach Date: Thu, 29 Aug 2024 15:37:54 +0500 Subject: [PATCH] refactoring TestIntegrationEmbedded, add getEmbeddedRestoreSettings to leverage https://github.com/ClickHouse/ClickHouse/issues/69053 --- pkg/backup/backuper.go | 32 +++++++++++++++---- pkg/backup/create.go | 2 +- pkg/backup/restore.go | 6 ++-- .../config-azblob-embedded-url.yml | 2 +- test/integration/integration_test.go | 28 +++++++++------- 5 files changed, 47 insertions(+), 23 deletions(-) diff --git a/pkg/backup/backuper.go b/pkg/backup/backuper.go index 361dc05f..c3ceb1e7 100644 --- a/pkg/backup/backuper.go +++ b/pkg/backup/backuper.go @@ -195,7 +195,8 @@ func (b *Backuper) isDiskTypeEncryptedObject(disk clickhouse.Disk, disks []click return underlyingIdx >= 0 } -func (b *Backuper) getEmbeddedBackupDefaultSettings(version int) []string { +// getEmbeddedRestoreSettings - different with getEmbeddedBackupSettings, cause https://github.com/ClickHouse/ClickHouse/issues/69053 +func (b *Backuper) getEmbeddedRestoreSettings(version int) []string { settings := []string{} if (b.cfg.General.RemoteStorage == "s3" || b.cfg.General.RemoteStorage == "gcs") && version >= 23007000 { settings = append(settings, "allow_s3_native_copy=1") @@ -209,7 +210,24 @@ func (b *Backuper) getEmbeddedBackupDefaultSettings(version int) []string { log.Fatal().Msgf("SET s3_use_adaptive_timeouts=0 error: %v", err) } } - if b.cfg.General.RemoteStorage == "azblob" && version >= 24005000 { + return settings +} + +func (b *Backuper) getEmbeddedBackupSettings(version int) []string { + settings := []string{} + if (b.cfg.General.RemoteStorage == "s3" || b.cfg.General.RemoteStorage == "gcs") && version >= 23007000 { + settings = append(settings, "allow_s3_native_copy=1") + if err := b.ch.Query("SET s3_request_timeout_ms=600000"); err != nil { + log.Fatal().Msgf("SET s3_request_timeout_ms=600000 error: %v", err) + } + + } + if (b.cfg.General.RemoteStorage == "s3" || b.cfg.General.RemoteStorage == "gcs") && version >= 23011000 { + if err := b.ch.Query("SET s3_use_adaptive_timeouts=0"); err != nil { + log.Fatal().Msgf("SET s3_use_adaptive_timeouts=0 error: %v", err) + } + } + if b.cfg.General.RemoteStorage == "azblob" && version >= 24005000 && b.cfg.ClickHouse.EmbeddedBackupDisk == "" { settings = append(settings, "allow_azure_native_copy=1") } return settings @@ -229,10 +247,10 @@ func (b *Backuper) getEmbeddedBackupLocation(ctx context.Context, backupName str return "", err } if b.cfg.S3.AccessKey != "" { - return fmt.Sprintf("S3('%s/%s','%s','%s')", s3Endpoint, backupName, b.cfg.S3.AccessKey, b.cfg.S3.SecretKey), nil + return fmt.Sprintf("S3('%s/%s/','%s','%s')", s3Endpoint, backupName, b.cfg.S3.AccessKey, b.cfg.S3.SecretKey), nil } if os.Getenv("AWS_ACCESS_KEY_ID") != "" { - return fmt.Sprintf("S3('%s/%s','%s','%s')", s3Endpoint, backupName, os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY")), nil + return fmt.Sprintf("S3('%s/%s/','%s','%s')", s3Endpoint, backupName, os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY")), nil } return "", fmt.Errorf("provide s3->access_key and s3->secret_key in config to allow embedded backup without `clickhouse->embedded_backup_disk`") } @@ -242,10 +260,10 @@ func (b *Backuper) getEmbeddedBackupLocation(ctx context.Context, backupName str return "", err } if b.cfg.GCS.EmbeddedAccessKey != "" { - return fmt.Sprintf("S3('%s/%s','%s','%s')", gcsEndpoint, backupName, b.cfg.GCS.EmbeddedAccessKey, b.cfg.GCS.EmbeddedSecretKey), nil + return fmt.Sprintf("S3('%s/%s/','%s','%s')", gcsEndpoint, backupName, b.cfg.GCS.EmbeddedAccessKey, b.cfg.GCS.EmbeddedSecretKey), nil } if os.Getenv("AWS_ACCESS_KEY_ID") != "" { - return fmt.Sprintf("S3('%s/%s','%s','%s')", gcsEndpoint, backupName, os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY")), nil + return fmt.Sprintf("S3('%s/%s/','%s','%s')", gcsEndpoint, backupName, os.Getenv("AWS_ACCESS_KEY_ID"), os.Getenv("AWS_SECRET_ACCESS_KEY")), nil } return "", fmt.Errorf("provide gcs->embedded_access_key and gcs->embedded_secret_key in config to allow embedded backup without `clickhouse->embedded_backup_disk`") } @@ -255,7 +273,7 @@ func (b *Backuper) getEmbeddedBackupLocation(ctx context.Context, backupName str return "", err } if b.cfg.AzureBlob.Container != "" { - return fmt.Sprintf("AzureBlobStorage('%s','%s','%s/%s')", azblobEndpoint, b.cfg.AzureBlob.Container, b.cfg.AzureBlob.ObjectDiskPath, backupName), nil + return fmt.Sprintf("AzureBlobStorage('%s','%s','%s/%s/')", azblobEndpoint, b.cfg.AzureBlob.Container, b.cfg.AzureBlob.ObjectDiskPath, backupName), nil } return "", fmt.Errorf("provide azblob->container and azblob->account_name, azblob->account_key in config to allow embedded backup without `clickhouse->embedded_backup_disk`") } diff --git a/pkg/backup/create.go b/pkg/backup/create.go index 62479909..28a2fca2 100644 --- a/pkg/backup/create.go +++ b/pkg/backup/create.go @@ -526,7 +526,7 @@ func (b *Backuper) generateEmbeddedBackupSQL(ctx context.Context, backupName str tablesSQL += ", " } } - backupSettings := b.getEmbeddedBackupDefaultSettings(version) + backupSettings := b.getEmbeddedBackupSettings(version) embeddedBackupLocation, err := b.getEmbeddedBackupLocation(ctx, backupName) if err != nil { return "", nil, err diff --git a/pkg/backup/restore.go b/pkg/backup/restore.go index b3de729d..48d22956 100644 --- a/pkg/backup/restore.go +++ b/pkg/backup/restore.go @@ -917,6 +917,7 @@ func (b *Backuper) fixEmbeddedMetadataRemote(ctx context.Context, backupName str } var fReader io.ReadCloser remoteFilePath := path.Join(objectDiskPath, backupName, "metadata", fInfo.Name()) + log.Debug().Msgf("read %s", remoteFilePath) fReader, err = b.dst.GetFileReaderAbsolute(ctx, path.Join(objectDiskPath, backupName, "metadata", fInfo.Name())) if err != nil { return err @@ -928,8 +929,9 @@ func (b *Backuper) fixEmbeddedMetadataRemote(ctx context.Context, backupName str } sqlQuery, sqlMetadataChanged, fixSqlErr := b.fixEmbeddedMetadataSQLQuery(ctx, sqlBytes, remoteFilePath, chVersion) if fixSqlErr != nil { - return fixSqlErr + return fmt.Errorf("b.fixEmbeddedMetadataSQLQuery return error: %v", fixSqlErr) } + log.Debug().Msgf("b.fixEmbeddedMetadataSQLQuery %s changed=%v", remoteFilePath, sqlMetadataChanged) if sqlMetadataChanged { err = b.dst.PutFileAbsolute(ctx, remoteFilePath, io.NopCloser(strings.NewReader(sqlQuery))) if err != nil { @@ -1653,7 +1655,7 @@ func (b *Backuper) restoreEmbedded(ctx context.Context, backupName string, schem } } } - settings := b.getEmbeddedBackupDefaultSettings(version) + settings := b.getEmbeddedRestoreSettings(version) if schemaOnly { settings = append(settings, "structure_only=1") } diff --git a/test/integration/config-azblob-embedded-url.yml b/test/integration/config-azblob-embedded-url.yml index d4b47c84..e235322c 100644 --- a/test/integration/config-azblob-embedded-url.yml +++ b/test/integration/config-azblob-embedded-url.yml @@ -28,7 +28,7 @@ azblob: endpoint_schema: http container: container1 object_disk_path: object_disk/{cluster}/{shard} - path: backup + path: backup/{cluster}/{shard} compression_format: none api: listen: :7171 diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index 5d291905..80d9aa42 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -638,16 +638,26 @@ func TestIntegrationEmbedded(t *testing.T) { t.Logf("@TODO RESTORE Ordinary with old syntax still not works for %s version, look https://github.com/ClickHouse/ClickHouse/issues/43971", os.Getenv("CLICKHOUSE_VERSION")) env, r := NewTestEnvironment(t) - //CUSTOM backup creates folder in each disk, need to clear - env.DockerExecNoError(r, "clickhouse", "rm", "-rfv", "/var/lib/clickhouse/disks/backups_s3/backup/") - env.runMainIntegrationScenario(t, "EMBEDDED_S3", "config-s3-embedded.yml") - + // === AZURE === // CUSTOM backup create folder in each disk env.DockerExecNoError(r, "clickhouse", "rm", "-rf", "/var/lib/clickhouse/disks/backups_azure/backup/") - env.runMainIntegrationScenario(t, "EMBEDDED_AZURE", "config-azblob-embedded.yml") - if compareVersion(version, "24.3") >= 0 { + if compareVersion(version, "24.8") >= 0 { env.runMainIntegrationScenario(t, "EMBEDDED_AZURE_URL", "config-azblob-embedded-url.yml") } + env.runMainIntegrationScenario(t, "EMBEDDED_AZURE", "config-azblob-embedded.yml") + + // === GCS over S3 === + if compareVersion(version, "24.3") >= 0 && os.Getenv("QA_GCS_OVER_S3_BUCKET") != "" { + //@todo think about named collections to avoid show credentials in logs look to https://github.com/fsouza/fake-gcs-server/issues/1330, https://github.com/fsouza/fake-gcs-server/pull/1164 + env.InstallDebIfNotExists(r, "clickhouse-backup", "ca-certificates", "gettext-base") + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "cat /etc/clickhouse-backup/config-gcs-embedded-url.yml.template | envsubst > /etc/clickhouse-backup/config-gcs-embedded-url.yml") + env.runMainIntegrationScenario(t, "EMBEDDED_GCS_URL", "config-gcs-embedded-url.yml") + } + + // === S3 === + // CUSTOM backup creates folder in each disk, need to clear + env.DockerExecNoError(r, "clickhouse", "rm", "-rfv", "/var/lib/clickhouse/disks/backups_s3/backup/") + env.runMainIntegrationScenario(t, "EMBEDDED_S3", "config-s3-embedded.yml") if compareVersion(version, "23.8") >= 0 { //CUSTOM backup creates folder in each disk, need to clear @@ -655,12 +665,6 @@ func TestIntegrationEmbedded(t *testing.T) { env.runMainIntegrationScenario(t, "EMBEDDED_LOCAL", "config-s3-embedded-local.yml") } if compareVersion(version, "24.3") >= 0 { - if os.Getenv("QA_GCS_OVER_S3_BUCKET") != "" { - //@todo think about named collections to avoid show credentials in logs look to https://github.com/fsouza/fake-gcs-server/issues/1330, https://github.com/fsouza/fake-gcs-server/pull/1164 - env.InstallDebIfNotExists(r, "clickhouse-backup", "ca-certificates", "gettext-base") - env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", "cat /etc/clickhouse-backup/config-gcs-embedded-url.yml.template | envsubst > /etc/clickhouse-backup/config-gcs-embedded-url.yml") - env.runMainIntegrationScenario(t, "EMBEDDED_GCS_URL", "config-gcs-embedded-url.yml") - } env.runMainIntegrationScenario(t, "EMBEDDED_S3_URL", "config-s3-embedded-url.yml") } //@TODO think about how to implements embedded backup for s3_plain disks