From c2f7b496e26a919e3219370dda9a32a835603b70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=9D=A8=E5=B8=86?= <39647285+leno23@users.noreply.github.com> Date: Sun, 17 May 2026 16:03:58 +0800 Subject: [PATCH 1/2] fix: skip missing remote table metadata Treat object-store metadata 404s as permanent so download skips the affected table immediately instead of exhausting the retry backoff. --- pkg/backup/download.go | 28 ++++++++++++++++++++++++++++ pkg/backup/download_test.go | 17 +++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/pkg/backup/download.go b/pkg/backup/download.go index c469e488a..69a808f77 100644 --- a/pkg/backup/download.go +++ b/pkg/backup/download.go @@ -40,6 +40,18 @@ var ( ErrBackupIsAlreadyExists = errors.New("backup is already exists") ) +func isRemoteMetadataNotFound(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(err.Error()) + return strings.Contains(message, "doesn't exist") || + strings.Contains(message, "key not found") || + strings.Contains(message, "nosuchkey") || + strings.Contains(message, "statuscode 404") || + strings.Contains(message, "statuscode: 404") +} + func (b *Backuper) Download(backupName string, tablePattern string, partitions []string, schemaOnly, rbacOnly, configsOnly, namedCollectionsOnly, resume bool, hardlinkExistsFiles bool, backupVersion string, commandId int) error { if pidCheckErr := pidlock.CheckAndCreatePidFile(backupName, "download"); pidCheckErr != nil { return errors.WithMessage(pidCheckErr, "CheckAndCreatePidFile") @@ -487,10 +499,15 @@ func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string, } } var tmBody []byte + metadataNotFound := false retry := retrier.New(retrier.ExponentialBackoff(b.cfg.General.RetriesOnFailure, common.AddRandomJitter(b.cfg.General.RetriesDuration, b.cfg.General.RetriesJitter)), b) err := retry.RunCtx(ctx, func(ctx context.Context) error { tmReader, err := b.dst.GetFileReader(ctx, remoteMetadataFile) if err != nil { + if isRemoteMetadataNotFound(err) { + metadataNotFound = true + return nil + } return errors.Wrapf(err, "can't GetFileReader(%s) error", remoteMetadataFile) } tmBody, err = io.ReadAll(tmReader) @@ -503,11 +520,22 @@ func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string, } return nil }) + // Missing metadata is permanent: do not burn retries, and skip the missing table. + if metadataNotFound { + logger.Warn().Str("remoteMetadataFile", remoteMetadataFile).Msg("metadata file not found on remote, skipping") + if strings.HasSuffix(localMetadataFile, ".json") { + return nil, size, nil + } + continue + } // sql file could be not present in incremental backup if err != nil && strings.HasSuffix(localMetadataFile, ".sql") { log.Warn().Str("localMetadataFile", localMetadataFile).Err(err).Send() continue } + if err != nil { + return nil, 0, err + } if err = os.MkdirAll(path.Dir(localMetadataFile), 0755); err != nil { return nil, 0, errors.WithMessage(err, "MkdirAll metadata dir") diff --git a/pkg/backup/download_test.go b/pkg/backup/download_test.go index 27ea7434e..a1d34a171 100644 --- a/pkg/backup/download_test.go +++ b/pkg/backup/download_test.go @@ -1,6 +1,7 @@ package backup import ( + "errors" "regexp" "testing" "time" @@ -91,6 +92,22 @@ var remoteBackup = storage.Backup{ UploadDate: time.Now(), } +func TestIsRemoteMetadataNotFound(t *testing.T) { + notFoundMessages := []string{ + "object doesn't exist", + "key not found: metadata/default/test.json", + "NoSuchKey: The specified key does not exist", + "operation error S3: GetObject, https response error StatusCode: 404", + "StatusCode 404", + } + for _, msg := range notFoundMessages { + assert.True(t, isRemoteMetadataNotFound(errors.New(msg)), msg) + } + + assert.False(t, isRemoteMetadataNotFound(nil)) + assert.False(t, isRemoteMetadataNotFound(errors.New("temporary network timeout"))) +} + func TestReBalanceTablesMetadataIfDiskNotExists_Files_NoErrors(t *testing.T) { remoteBackup.DataFormat = "tar" baseTable := metadata.TableMetadata{ From 4725011040dfbfc190389f894fa23e82fd27b2e6 Mon Sep 17 00:00:00 2001 From: slach Date: Sat, 30 May 2026 17:47:49 +0400 Subject: [PATCH 2/2] fix: fail fast on missing remote table metadata during download A missing table .json on remote storage is a permanent broken-backup condition, not a transient one. Return a clear error instead of silently skipping the table (a silent skip restores fewer tables than the backup claims, with no signal to the operator). Detection now covers all backends' not-found phrasings (S3 NoSuchKey, GCS doesn't exist/404, Azure BlobNotFound, FTP "No such file or directory", SFTP "file does not exist") so the 404 breaks out of the retry loop immediately instead of burning the ~35s exponential backoff. Add per-backend integration coverage (S3/SFTP/FTP/GCS/GCS-emulator/ AZBLOB/COS) asserting download fails fast with a not-found error and without retrying. Co-Authored-By: Claude Opus 4.8 (1M context) --- pkg/backup/download.go | 33 +- pkg/backup/download_test.go | 4 + test/integration/testMetadataNotFound_test.go | 366 ++++++++++++++++++ 3 files changed, 395 insertions(+), 8 deletions(-) create mode 100644 test/integration/testMetadataNotFound_test.go diff --git a/pkg/backup/download.go b/pkg/backup/download.go index 33f1b512d..2436c0213 100644 --- a/pkg/backup/download.go +++ b/pkg/backup/download.go @@ -57,11 +57,24 @@ func isRemoteMetadataNotFound(err error) bool { return false } message := strings.ToLower(err.Error()) - return strings.Contains(message, "doesn't exist") || - strings.Contains(message, "key not found") || - strings.Contains(message, "nosuchkey") || - strings.Contains(message, "statuscode 404") || - strings.Contains(message, "statuscode: 404") + // Every remote storage backend phrases "object is missing" differently, so we + // match the known permanent-not-found markers across S3/GCS/Azure/FTP/SFTP/FS. + for _, marker := range []string{ + "doesn't exist", // GCS + "does not exist", // SFTP ("file does not exist"), Azure ("the specified blob does not exist") + "no such file or directory", // FTP (550), local filesystem + "key not found", + "nosuchkey", // S3 + "blobnotfound", // Azure Blob (x-ms-error-code) + "statuscode 404", // S3 SDK v2 + "statuscode: 404", + "status: 404", // Azure ("RESPONSE Status: 404") + } { + if strings.Contains(message, marker) { + return true + } + } + return false } func (b *Backuper) Download(backupName string, tablePattern string, partitions []string, schemaOnly, rbacOnly, configsOnly, namedCollectionsOnly, resume bool, hardlinkExistsFiles bool, backupVersion string, commandId int) error { @@ -530,12 +543,16 @@ func (b *Backuper) downloadTableMetadata(ctx context.Context, backupName string, } return nil }) - // Missing metadata is permanent: do not burn retries, and skip the missing table. + // Missing metadata is permanent: a 404 will never become available, so we + // detect it inside the retry closure and break out without burning the + // exponential backoff. A missing table .json means the backup is broken, + // so fail fast with a clear error. The optional .sql (incremental/embedded + // metadata) may legitimately be absent, so it is still skipped. if metadataNotFound { - logger.Warn().Str("remoteMetadataFile", remoteMetadataFile).Msg("metadata file not found on remote, skipping") if strings.HasSuffix(localMetadataFile, ".json") { - return nil, size, nil + return nil, 0, errors.Errorf("remote metadata file %s not found on remote storage, backup is broken", remoteMetadataFile) } + logger.Warn().Str("remoteMetadataFile", remoteMetadataFile).Msg("metadata file not found on remote, skipping") continue } // sql file could be not present in incremental backup diff --git a/pkg/backup/download_test.go b/pkg/backup/download_test.go index 9d18bdc81..d8df34b33 100644 --- a/pkg/backup/download_test.go +++ b/pkg/backup/download_test.go @@ -102,6 +102,10 @@ func TestIsRemoteMetadataNotFound(t *testing.T) { "NoSuchKey: The specified key does not exist", "operation error S3: GetObject, https response error StatusCode: 404", "StatusCode 404", + // real backend phrasings observed in test/integration TestMetadataNotFound* + "550 /backup/metadata/default/test.json: No such file or directory", // FTP + "file does not exist", // SFTP + "AzureBlob GetFileReaderAbsolute Download: RESPONSE ERROR (ServiceCode=BlobNotFound) RESPONSE Status: 404 The specified blob does not exist.", // Azure } for _, msg := range notFoundMessages { assert.True(t, isRemoteMetadataNotFound(errors.New(msg)), msg) diff --git a/test/integration/testMetadataNotFound_test.go b/test/integration/testMetadataNotFound_test.go new file mode 100644 index 000000000..0d65db655 --- /dev/null +++ b/test/integration/testMetadataNotFound_test.go @@ -0,0 +1,366 @@ +//go:build integration + +package main + +import ( + "context" + "fmt" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/Altinity/clickhouse-backup/v2/pkg/utils" + + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" +) + +// Each TestMetadataNotFound* function reproduces https://github.com/Altinity/clickhouse-backup/issues/1379: +// - create a backup with a single table and upload it, +// - delete that table's metadata//.json on remote storage, +// - run `download` and assert it fails fast with a "not found" error. +// +// "Fast" is the key check: a 404 on the metadata file is permanent, so the +// retry loop must break out immediately instead of burning the exponential +// backoff (RetriesOnFailure=3 at 5s base ≈ 35s). A failure well under 20s +// proves the retry loop was skipped. +// +// Each backend is its own top-level test so they can be run independently +// (e.g. `RUN_TESTS=TestMetadataNotFoundS3 ./test/integration/run.sh`). +// Backends that need cloud credentials skip themselves when the corresponding +// env var (GCS_TESTS, AZURE_TESTS, QA_TENCENT_SECRET_KEY/QA_TENCENT_SECRET_ID) +// is unset. + +// metadataNotFoundCase wires one remote-storage backend to the shared scenario. +// +// root is derived in runMetadataNotFoundScenario from configFile: +// env.resolveConfigPaths reads `path` from the YAML and asks ClickHouse to +// expand {cluster}/{shard}/{version} via ApplyMacros. pathPrefix is prepended — +// used by backends where the config path is logical (mc alias, ftp chroot home) +// rather than physical. file is the metadata json key relative to `path`, +// i.e. "/metadata//
.json". +// +// When fsContainer is non-empty, runMetadataNotFoundScenario will `mkdir -p root` +// inside that container before the backup runs. +type metadataNotFoundCase struct { + name string + configFile string + pathPrefix string + fsContainer string + skip func() bool + skipReason string + setup func(env *TestEnvironment, r *require.Assertions) + // assertExists verifies the metadata json is present on remote before deletion. + assertExists func(env *TestEnvironment, r *require.Assertions, root, file string) + // deleteFile removes the single metadata json at root/file on remote storage. + deleteFile func(env *TestEnvironment, r *require.Assertions, root, file string) +} + +func TestMetadataNotFoundS3(t *testing.T) { + runMetadataNotFoundCase(t, s3MetadataNotFoundCase()) +} +func TestMetadataNotFoundSFTP(t *testing.T) { + runMetadataNotFoundCase(t, sftpMetadataNotFoundCase()) +} +func TestMetadataNotFoundFTP(t *testing.T) { + runMetadataNotFoundCase(t, ftpMetadataNotFoundCase()) +} +func TestMetadataNotFoundGCSEmulator(t *testing.T) { + runMetadataNotFoundCase(t, gcsEmulatorMetadataNotFoundCase()) +} +func TestMetadataNotFoundAZBLOB(t *testing.T) { + runMetadataNotFoundCase(t, azblobMetadataNotFoundCase()) +} +func TestMetadataNotFoundGCS(t *testing.T) { + runMetadataNotFoundCase(t, gcsRealMetadataNotFoundCase()) +} +func TestMetadataNotFoundCOS(t *testing.T) { + runMetadataNotFoundCase(t, cosMetadataNotFoundCase()) +} + +func runMetadataNotFoundCase(t *testing.T, tc metadataNotFoundCase) { + if tc.skip != nil && tc.skip() { + t.Skip(tc.skipReason) + return + } + runMetadataNotFoundScenario(t, tc) +} + +func runMetadataNotFoundScenario(t *testing.T, tc metadataNotFoundCase) { + chVer := strings.ReplaceAll(os.Getenv("CLICKHOUSE_VERSION"), ".", "_") + + env, r := NewTestEnvironment(t) + env.connectWithWait(t, r, 0*time.Second, 1*time.Second, 1*time.Minute) + defer env.Cleanup(t, r) + + r.NoError(env.DockerCP("configs/"+tc.configFile, "clickhouse-backup:/etc/clickhouse-backup/config.yml")) + + cfgPath, _ := env.resolveConfigPaths(r, tc.configFile) + root := tc.pathPrefix + cfgPath + + if tc.fsContainer != "" { + env.DockerExecNoError(r, tc.fsContainer, "mkdir", "-p", root) + } + if tc.setup != nil { + tc.setup(env, r) + } + + suffix := time.Now().UnixNano() + tableShort := fmt.Sprintf("test_metadata_not_found_%s", strings.ToLower(tc.name)) + tableName := "default." + tableShort + backupName := fmt.Sprintf("metadata_not_found_%s_%d", chVer, suffix) + // Remote metadata json key relative to the configured `path`. Plain ASCII + // db/table names are encoded as-is by common.TablePathEncode. + metaFile := fmt.Sprintf("%s/metadata/default/%s.json", backupName, tableShort) + + env.queryWithNoError(r, fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s(id UInt64) ENGINE=MergeTree() ORDER BY id", tableName)) + env.queryWithNoError(r, fmt.Sprintf("INSERT INTO %s SELECT number FROM numbers(100)", tableName)) + t.Cleanup(func() { + dropQ := "DROP TABLE IF EXISTS " + tableName + if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "20.3") > 0 { + dropQ += " NO DELAY" + } + if _, err := env.DockerExecOut("clickhouse", "clickhouse", "client", "-q", dropQ); err != nil { + log.Warn().Err(err).Str("table", tableName).Msg("t.Cleanup: failed to drop table") + } + }) + + // Best-effort teardown so a mid-test failure does not leak the backup into a + // shared bucket. download is expected to fail, so it may leave a partial + // local backup — delete local first, ignore errors. + defer func() { + for _, cmd := range [][]string{ + {"clickhouse-backup", "delete", "local", backupName}, + {"clickhouse-backup", "delete", "remote", backupName}, + } { + if out, err := env.DockerExecOut("clickhouse-backup", cmd...); err != nil { + log.Warn().Err(err).Str("cmd", strings.Join(cmd, " ")).Msgf("metadataNotFound teardown: %s", out) + } + } + }() + + log.Debug().Str("backend", tc.name).Msg("Create and upload a single-table backup") + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "create", "--tables", tableName, backupName) + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "upload", backupName) + // Drop the local copy so download must read metadata from remote. + env.DockerExecNoError(r, "clickhouse-backup", "clickhouse-backup", "delete", "local", backupName) + + log.Debug().Str("backend", tc.name).Msg("Delete the table metadata json on remote storage") + tc.assertExists(env, r, root, metaFile) + tc.deleteFile(env, r, root, metaFile) + + log.Debug().Str("backend", tc.name).Msg("Download must fail fast with a not-found error (no retry backoff)") + start := time.Now() + out, err := env.DockerExecOut("clickhouse-backup", "clickhouse-backup", "download", backupName) + elapsed := time.Since(start) + log.Debug().Msg(out) + r.Error(err, "download must fail when remote table metadata is missing, output: %s", out) + // The fail-fast path emits this distinctive message; the raw backend errors + // (e.g. "no such file or directory", "BlobNotFound") do not contain it, so a + // match proves isRemoteMetadataNotFound classified the 404 correctly. + r.Contains(strings.ToLower(out), "not found on remote storage", "download must report the missing metadata as not-found, output: %s", out) + // "Will wait near Ns and retry" (pkg/backup/backuper.go) is logged on every + // retry attempt; its absence proves the permanent 404 broke out immediately. + r.NotContains(out, "and retry", "download must not retry on a permanent 404, output: %s", out) + // Defense in depth: RetriesOnFailure=3 with 5s exponential backoff would add + // ~35s (5+10+20); a fast failure confirms the retry loop was skipped. + r.Less(elapsed, 20*time.Second, "download must not burn the retry backoff on a permanent 404, took %s, output: %s", elapsed, out) +} + +// containerFSMetadataCase builds a case for a backend that maps its remote +// storage to a path on the given docker container's filesystem. +func containerFSMetadataCase(name, configFile, container string) metadataNotFoundCase { + return metadataNotFoundCase{ + name: name, + configFile: configFile, + fsContainer: container, + assertExists: func(env *TestEnvironment, r *require.Assertions, root, file string) { + out, err := env.DockerExecOut(container, "ls", "-l", root+"/"+file) + r.NoError(err, "expected %s/%s to exist on %s, output: %s", root, file, container, out) + }, + deleteFile: func(env *TestEnvironment, r *require.Assertions, root, file string) { + env.DockerExecNoError(r, container, "rm", "-f", root+"/"+file) + }, + } +} + +func s3MetadataNotFoundCase() metadataNotFoundCase { + // MinIO only sees objects that went through its S3 API, so use `mc`. + const mcAliasCmd = "mc alias set local https://localhost:9000 access_key it_is_my_super_secret_key >/dev/null 2>&1" + return metadataNotFoundCase{ + name: "S3", + configFile: "config-s3.yml", + pathPrefix: "local/clickhouse/", // mc alias + bucket; config path is relative to bucket + assertExists: func(env *TestEnvironment, r *require.Assertions, root, file string) { + out, err := env.DockerExecOut("minio", "bash", "-c", fmt.Sprintf("%s && mc stat %s/%s", mcAliasCmd, root, file)) + r.NoError(err, "expected %s/%s to exist, got: %s", root, file, out) + }, + deleteFile: func(env *TestEnvironment, r *require.Assertions, root, file string) { + env.DockerExecNoError(r, "minio", "bash", "-c", fmt.Sprintf("%s && mc rm %s/%s", mcAliasCmd, root, file)) + }, + } +} + +func sftpMetadataNotFoundCase() metadataNotFoundCase { + tc := containerFSMetadataCase("SFTP", "config-sftp-auth-key.yaml", "sshd") + tc.setup = func(env *TestEnvironment, r *require.Assertions) { + env.uploadSSHKeys(r, "clickhouse-backup") + } + return tc +} + +func ftpMetadataNotFoundCase() metadataNotFoundCase { + home := "/home/test_backup" + if isAdvancedMode() { + home = "/home/ftpusers/test_backup" + } + tc := containerFSMetadataCase("FTP", "config-ftp.yaml", "ftp") + // FTP server chroots users to `home`; config paths like `/backup` resolve to + // `/backup` on the container filesystem. + tc.pathPrefix = home + tc.skip = func() bool { return compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.8") <= 0 } + tc.skipReason = "FTP scenario only validated on ClickHouse > 21.8" + tc.setup = func(env *TestEnvironment, r *require.Assertions) { + // proftpd/vsftpd containers don't create `test_backup` as a system user; uid 1000 owns the home dir. + env.DockerExecNoError(r, "ftp", "sh", "-c", fmt.Sprintf("chown -R 1000:1000 %s && chmod -R 0777 %s", home, home)) + } + return tc +} + +func gcsEmulatorMetadataNotFoundCase() metadataNotFoundCase { + const bucket = "altinity-qa-test" + const baseURL = "http://localhost:8080" + // fake-gcs-server addresses single objects at /o/, with + // slashes escaped as %2F (url.QueryEscape). + objURL := func(root, file string) string { + return fmt.Sprintf("%s/storage/v1/b/%s/o/%s", baseURL, bucket, url.QueryEscape(root+"/"+file)) + } + return metadataNotFoundCase{ + name: "GCS_EMULATOR", + configFile: "config-gcs-custom-endpoint.yml", + setup: func(env *TestEnvironment, r *require.Assertions) { + env.DockerExecNoError(r, "gcs", "apk", "add", "-q", "curl") + }, + assertExists: func(env *TestEnvironment, r *require.Assertions, root, file string) { + out, err := env.DockerExecOut("gcs", "sh", "-c", fmt.Sprintf(`curl -g -s "%s"`, objURL(root, file))) + r.NoError(err, "assertExists curl failed: %s", out) + r.Contains(out, `"name"`, "expected object metadata for %s/%s, got: %s", root, file, out) + }, + deleteFile: func(env *TestEnvironment, r *require.Assertions, root, file string) { + env.DockerExecNoError(r, "gcs", "sh", "-c", fmt.Sprintf(`curl -g -s -X DELETE "%s"`, objURL(root, file))) + }, + } +} + +func azblobMetadataNotFoundCase() metadataNotFoundCase { + const container = "container1" + const accountName = "devstoreaccount1" + const accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + const azureCliImage = "mcr.microsoft.com/azure-cli:latest" + azConnString := fmt.Sprintf( + "DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=http://azure:10000/%s;", + accountName, accountKey, accountName, + ) + azRun := func(env *TestEnvironment, args ...string) (string, error) { + dockerArgs := append([]string{ + "run", "--rm", "--network", env.tc.networkName, + "-e", "AZURE_STORAGE_CONNECTION_STRING=" + azConnString, + azureCliImage, "az", + }, args...) + return utils.ExecCmdOut(context.Background(), dockerExecTimeout, "docker", dockerArgs...) + } + blobName := func(root, file string) string { return strings.TrimPrefix(root+"/"+file, "/") } + return metadataNotFoundCase{ + name: "AZBLOB", + configFile: "config-azblob.yml", + skip: func() bool { return isTestShouldSkip("AZURE_TESTS") }, + skipReason: "Skipping AZBLOB integration tests (AZURE_TESTS not set)", + setup: func(env *TestEnvironment, r *require.Assertions) { + env.tc.pullImageIfNeeded(context.Background(), azureCliImage) + }, + assertExists: func(env *TestEnvironment, r *require.Assertions, root, file string) { + out, err := azRun(env, "storage", "blob", "show", "--container-name", container, "--name", blobName(root, file)) + r.NoError(err, "expected azblob %s to exist, got: %s", blobName(root, file), out) + }, + deleteFile: func(env *TestEnvironment, r *require.Assertions, root, file string) { + out, err := azRun(env, "storage", "blob", "delete", "--container-name", container, "--name", blobName(root, file)) + r.NoError(err, "azblob delete %s: %s", blobName(root, file), out) + }, + } +} + +func gcsRealMetadataNotFoundCase() metadataNotFoundCase { + const bucket = "altinity-qa-test" + const image = "gcr.io/google.com/cloudsdktool/google-cloud-cli:slim" + const authPrefix = "gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS >/dev/null 2>&1 && " + gsutil := func(env *TestEnvironment, r *require.Assertions, sh string) string { + args := []string{ + "run", "--rm", "--network", env.tc.networkName, + "--volumes-from", env.tc.GetContainerID("clickhouse-backup"), + "-e", "GOOGLE_APPLICATION_CREDENTIALS=/etc/clickhouse-backup/credentials.json", + image, "bash", "-c", authPrefix + sh, + } + out, err := utils.ExecCmdOut(context.Background(), dockerExecTimeout, "docker", args...) + r.NoError(err, "gsutil command `%s` failed: %s", sh, out) + return out + } + return metadataNotFoundCase{ + name: "GCS", + configFile: "config-gcs.yml", + skip: func() bool { return isTestShouldSkip("GCS_TESTS") }, + skipReason: "Skipping GCS integration tests (GCS_TESTS not set)", + setup: func(env *TestEnvironment, r *require.Assertions) { + env.tc.pullImageIfNeeded(context.Background(), image) + }, + assertExists: func(env *TestEnvironment, r *require.Assertions, root, file string) { + out := gsutil(env, r, fmt.Sprintf("gsutil ls gs://%s/%s/%s", bucket, root, file)) + r.Contains(out, "gs://"+bucket+"/"+root+"/"+file, "expected listing to contain %s/%s", root, file) + }, + deleteFile: func(env *TestEnvironment, r *require.Assertions, root, file string) { + gsutil(env, r, fmt.Sprintf("gsutil -q rm gs://%s/%s/%s", bucket, root, file)) + }, + } +} + +func cosMetadataNotFoundCase() metadataNotFoundCase { + // COS exposes an S3-compatible API on its regional endpoint. + const bucket = "clickhouse-backup-1336113806" + const endpoint = "https://cos.na-ashburn.myqcloud.com" + const image = "amazon/aws-cli:latest" + // Tencent COS rejects path-style addressing; force virtual-hosted style. + const awsPrefix = "aws configure set default.s3.addressing_style virtual >/dev/null && " + awsRun := func(env *TestEnvironment, r *require.Assertions, sh string) string { + out, err := utils.ExecCmdOut(context.Background(), dockerExecTimeout, "docker", + "run", "--rm", "--network", env.tc.networkName, + "-e", "AWS_ACCESS_KEY_ID="+os.Getenv("QA_TENCENT_SECRET_ID"), + "-e", "AWS_SECRET_ACCESS_KEY="+os.Getenv("QA_TENCENT_SECRET_KEY"), + "-e", "AWS_DEFAULT_REGION=na-ashburn", + "--entrypoint", "sh", image, "-c", awsPrefix+sh) + r.NoError(err, "aws-cli failed: %s", out) + return out + } + return metadataNotFoundCase{ + name: "COS", + configFile: "config-cos.yml", + skip: func() bool { + return os.Getenv("QA_TENCENT_SECRET_KEY") == "" || os.Getenv("QA_TENCENT_SECRET_ID") == "" + }, + skipReason: "Skipping COS integration tests (QA_TENCENT_SECRET_ID / QA_TENCENT_SECRET_KEY not set)", + setup: func(env *TestEnvironment, r *require.Assertions) { + env.tc.pullImageIfNeeded(context.Background(), image) + env.InstallDebIfNotExists(r, "clickhouse-backup", "gettext-base") + // config.yml was copied raw and still has ${QA_TENCENT_SECRET_*} placeholders. + env.DockerExecNoError(r, "clickhouse-backup", "bash", "-xec", + "envsubst < /etc/clickhouse-backup/config.yml > /tmp/c.yml && mv /tmp/c.yml /etc/clickhouse-backup/config.yml") + }, + assertExists: func(env *TestEnvironment, r *require.Assertions, root, file string) { + out := awsRun(env, r, fmt.Sprintf("aws --endpoint-url=%s s3 ls s3://%s/%s/%s", endpoint, bucket, root, file)) + r.Contains(out, file[strings.LastIndex(file, "/")+1:], "expected %s/%s on COS, got: %s", root, file, out) + }, + deleteFile: func(env *TestEnvironment, r *require.Assertions, root, file string) { + awsRun(env, r, fmt.Sprintf("aws --endpoint-url=%s s3 rm s3://%s/%s/%s", endpoint, bucket, root, file)) + }, + } +}