From ac2387ed0210d4ca15ec22764d2c9827c30b72ec Mon Sep 17 00:00:00 2001 From: Aditya Maru Date: Mon, 25 Apr 2022 09:46:33 -0400 Subject: [PATCH] gcp: make per-chunk retry upload timeout configurable This change adds a cluster setting `cloudstorage.gs.chunking.retry_timeout` that can be used to change the default per-chunk retry timeout that GCS imposes when chunking of file upload is enabled. The default value is set to 60 seconds, which is double of the default google sdk value of 30s. This change was motivated by sporadic occurrences of a 503 service unavailable error during backups. On its own this change is not expected to solve the resiliency issues of backup when the upload service is unavailable, but it is nice to have configurable setting nonetheless. Release note (sql change): `cloudstorage.gs.chunking.retry_timeout` is a cluster setting that can be used to configure the per-chunk retry timeout of files to Google Cloud Storage. The default value is 60 seconds. --- pkg/cloud/gcp/gcs_storage.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/cloud/gcp/gcs_storage.go b/pkg/cloud/gcp/gcs_storage.go index b9d9c263a9fa..62ba988508c7 100644 --- a/pkg/cloud/gcp/gcs_storage.go +++ b/pkg/cloud/gcp/gcs_storage.go @@ -66,6 +66,15 @@ var gcsChunkingEnabled = settings.RegisterBoolSetting( true, /* default */ ) +// gcsChunkRetryTimeout is used to configure the per-chunk retry deadline when +// uploading chunks to Google Cloud Storage. +var gcsChunkRetryTimeout = settings.RegisterDurationSetting( + settings.TenantWritable, + "cloudstorage.gs.chunking.retry_timeout", + "per-chunk retry deadline when chunking of file upload to Google Cloud Storage", + 60, +) + func parseGSURL(_ cloud.ExternalStorageURIContext, uri *url.URL) (cloudpb.ExternalStorage, error) { gsURL := cloud.ConsumeURL{URL: uri} conf := cloudpb.ExternalStorage{} @@ -253,6 +262,7 @@ func (g *gcsStorage) Writer(ctx context.Context, basename string) (io.WriteClose if !gcsChunkingEnabled.Get(&g.settings.SV) { w.ChunkSize = 0 } + w.ChunkRetryDeadline = gcsChunkRetryTimeout.Get(&g.settings.SV) return w, nil }