diff --git a/CHANGELOG.md b/CHANGELOG.md index 5722c256a4..b144d46e19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Added - [#5337](https://github.com/thanos-io/thanos/pull/5337) Thanos Object Store: Add the `prefix` option to buckets +- [#5409](https://github.com/thanos-io/thanos/pull/5409) S3: Add option to force DNS style lookup. - [#5352](https://github.com/thanos-io/thanos/pull/5352) Cache: Add cache metrics to groupcache. - [#5391](https://github.com/thanos-io/thanos/pull/5391) Receive: Add relabeling support. - [#5408](https://github.com/thanos-io/thanos/pull/5391) Receive: Add support for consistent hashrings. diff --git a/docs/storage.md b/docs/storage.md index 47f2bd3f19..b2f0316537 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -89,6 +89,7 @@ config: trace: enable: false list_objects_version: "" + bucket_lookup_type: auto part_size: 67108864 sse_config: type: "" @@ -119,6 +120,8 @@ Set `list_objects_version: "v1"` for S3 compatible APIs that don't support ListO `http_config.tls_config` allows configuring TLS connections. Please refer to the document of [tls_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tls_config) for detailed information on what each option does. +`bucket_lookup_type` can be `auto`, `virtual-hosted` or `path`. Read more about it [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). + For debug and testing purposes you can set * `insecure: true` to switch to plain insecure HTTP instead of HTTPS diff --git a/pkg/objstore/s3/s3.go b/pkg/objstore/s3/s3.go index 1ece1a51f8..1e0ae3fe62 100644 --- a/pkg/objstore/s3/s3.go +++ b/pkg/objstore/s3/s3.go @@ -34,7 +34,50 @@ import ( type ctxKey int +type BucketLookupType int + +func (blt BucketLookupType) String() string { + return []string{"auto", "virtual-hosted", "path"}[blt] +} + +func (blt BucketLookupType) MinioType() minio.BucketLookupType { + return []minio.BucketLookupType{ + minio.BucketLookupAuto, + minio.BucketLookupDNS, + minio.BucketLookupPath, + }[blt] +} + +func (blt BucketLookupType) MarshalYAML() (interface{}, error) { + return blt.String(), nil +} + +func (blt *BucketLookupType) UnmarshalYAML(unmarshal func(interface{}) error) error { + var lookupType string + if err := unmarshal(&lookupType); err != nil { + return err + } + + switch lookupType { + case "auto": + *blt = AutoLookup + return nil + case "virtual-hosted": + *blt = VirtualHostLookup + return nil + case "path": + *blt = PathLookup + return nil + } + + return fmt.Errorf("unsupported bucket lookup type: %s", lookupType) +} + const ( + AutoLookup BucketLookupType = iota + VirtualHostLookup + PathLookup + // DirDelim is the delimiter used to model a directory structure in an object store bucket. DirDelim = "/" @@ -66,7 +109,8 @@ var DefaultConfig = Config{ MaxIdleConnsPerHost: 100, MaxConnsPerHost: 0, }, - PartSize: 1024 * 1024 * 64, // 64MB. + PartSize: 1024 * 1024 * 64, // 64MB. + BucketLookupType: AutoLookup, } // Config stores the configuration for s3 bucket. @@ -83,6 +127,7 @@ type Config struct { HTTPConfig HTTPConfig `yaml:"http_config"` TraceConfig TraceConfig `yaml:"trace"` ListObjectsVersion string `yaml:"list_objects_version"` + BucketLookupType BucketLookupType `yaml:"bucket_lookup_type"` // PartSize used for multipart upload. Only used if uploaded object size is known and larger than configured PartSize. // NOTE we need to make sure this number does not produce more parts than 10 000. PartSize uint64 `yaml:"part_size"` @@ -265,10 +310,11 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B } client, err := minio.New(config.Endpoint, &minio.Options{ - Creds: credentials.NewChainCredentials(chain), - Secure: !config.Insecure, - Region: config.Region, - Transport: rt, + Creds: credentials.NewChainCredentials(chain), + Secure: !config.Insecure, + Region: config.Region, + Transport: rt, + BucketLookup: config.BucketLookupType.MinioType(), }) if err != nil { return nil, errors.Wrap(err, "initialize s3 client") diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 587d9e05c3..a2d62f8f09 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -959,6 +959,7 @@ func NewS3Config(bucket, endpoint, basePath string) s3.Config { KeyFile: filepath.Join(basePath, "certs", "private.key"), }, }, + BucketLookupType: s3.AutoLookup, } }