final polishing BACKUP/RESTORE for S3/AzureBlobStorage, fix #695, wai…

…t when XML-API will implement on fsouza/fake-gcs-server#1164, fsouza/fake-gcs-server#1330
Altinity · Feb 28, 2024 · 25357bb · 25357bb
1 parent a947258
commit 25357bb
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 2 deletions.
diff --git a/pkg/backup/restore.go b/pkg/backup/restore.go
@@ -529,7 +529,6 @@ func (b *Backuper) restoreSchemaEmbedded(ctx context.Context, backupName string,
 	if err != nil {
 		return err
 	}
-	/*}*/
 	return b.restoreEmbedded(ctx, backupName, true, tablesForRestore, nil)
 }
 

diff --git a/test/integration/config-gcs-embedded-url.yml b/test/integration/config-gcs-embedded-url.yml
@@ -0,0 +1,20 @@
+general:
+  disable_progress_bar: true
+  remote_storage: gcs
+  upload_concurrency: 4
+  download_concurrency: 4
+  restore_schema_on_cluster: "{cluster}"
+clickhouse:
+  host: clickhouse
+  port: 9000
+  restart_command: bash -c 'echo "FAKE RESTART"'
+  use_embedded_backup_restore: true
+  timeout: 4h
+gcs:
+  bucket: "${QA_GCS_OVER_S3_BUCKET}"
+  path: backup/{cluster}/{shard}
+  object_disk_path: object_disks/{cluster}/{shard}
+  credentials_file: /etc/clickhouse-backup/credentials.json
+  embedded_access_key: "${QA_GCS_OVER_S3_ACCESS_KEY}"
+  embedded_secret_key: "${QA_GCS_OVER_S3_SECRET_KEY}"
+  compression_format: none
diff --git a/test/integration/config-gcs.yml b/test/integration/config-gcs.yml
@@ -14,4 +14,3 @@ gcs:
   object_disk_path: object_disks/{cluster}/{shard}
   credentials_file: /etc/clickhouse-backup/credentials.json
   compression_format: tar
-#  endpoint: http://gcs:8080/storage/v1/
diff --git a/test/integration/docker-compose_advanced.yml b/test/integration/docker-compose_advanced.yml
@@ -184,6 +184,10 @@ services:
       QA_AWS_SECRET_KEY: ${QA_AWS_SECRET_KEY}
       QA_AWS_BUCKET: ${QA_AWS_BUCKET}
       QA_AWS_REGION: ${QA_AWS_REGION}
+# GCS over S3 embedded backups
+      QA_GCS_OVER_S3_ACCESS_KEY: "${QA_GCS_OVER_S3_ACCESS_KEY}"
+      QA_GCS_OVER_S3_SECRET_KEY: "${QA_GCS_OVER_S3_SECRET_KEY}"
+      QA_GCS_OVER_S3_BUCKET: "${QA_GCS_OVER_S3_BUCKET}"
 # https://github.com/Altinity/clickhouse-backup/issues/691:
       AWS_ACCESS_KEY_ID: access_key
       AWS_SECRET_ACCESS_KEY: it_is_my_super_secret_key
@@ -252,6 +256,7 @@ services:
       - ./config-ftp.yaml:/etc/clickhouse-backup/config-ftp.yaml
       - ./config-ftp-old.yaml:/etc/clickhouse-backup/config-ftp-old.yaml
       - ./config-gcs.yml:/etc/clickhouse-backup/config-gcs.yml
+      - ./config-gcs-embedded-url.yml:/etc/clickhouse-backup/config-gcs-embedded-url.yml.template
       - ./config-gcs-custom-endpoint.yml:/etc/clickhouse-backup/config-gcs-custom-endpoint.yml
       - ./config-s3.yml:/etc/clickhouse-backup/config-s3.yml
       - ./config-s3-embedded.yml:/etc/clickhouse-backup/config-s3-embedded.yml

diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go
@@ -1832,6 +1832,10 @@ func TestIntegrationEmbedded(t *testing.T) {
 	//r.NoError(dockerCP("azure:/tmp/azurite_http.pcap", "./azurite_http.pcap"))
 
 	if compareVersion(version, "23.8") >= 0 {
+		//@todo think about named collections to avoid show credentials in logs look to https://github.com/fsouza/fake-gcs-server/issues/1330
+		//installDebIfNotExists(r, "clickhouse-backup", "ca-certificates", "gettext-base")
+		//r.NoError(dockerExec("clickhouse-backup", "bash", "-xec", "cat /etc/clickhouse-backup/config-gcs-embedded-url.yml.template | envsubst > /etc/clickhouse-backup/config-gcs-embedded-url.yml"))
+		//runMainIntegrationScenario(t, "EMBEDDED_GCS_URL", "config-gcs-embedded-url.yml")
 		runMainIntegrationScenario(t, "EMBEDDED_S3_URL", "config-s3-embedded-url.yml")
 	}
 	//CUSTOM backup create folder in each disk