Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
sidecar/compact/store/receiver - Add the prefix option to buckets (#5337
) * Create prefixed bucket Signed-off-by: jademcosta <jade.costa@nubank.com.br> * started PrefixedBucket tests Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * finish objstore tests Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * Simplify string removal logic Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Test more prefix cases on PrefixedBucket Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Only use a prefixedbucket if we have a valid prefix Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Add single unit test for prefixedBucket prefix Signed-off-by: jademcosta <jade.costa@nubank.com.br> * test other prefixes on UsesPrefixTest Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * add remaining methods to UsesPrefixTest Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * add prefix to docs examples Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * Simplify Iter method Signed-off-by: jademcosta <jade.costa@nubank.com.br> * add prefix explanation to S3 docs Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * Conclusion of prefix sentence on docs Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Use DirDelim instead of magic string Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Add log when using prefixed bucket Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Remove "@" from test string to make them simpler Signed-off-by: jademcosta <jade.costa@nubank.com.br> * fix BucketConfig Config type - back to interface Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * add changelog Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * add missing checks in UsesPrefixTest Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * fix linter and test errors Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * Add license to new files Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Remove autogenerated docs Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Remove duplicated transformation of string->[]byte Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Add prefixed bucket on all e2e tests for S3 The idea is that if it works, we can add for all other providers. Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Add e2e tests using prefixed bucket to all providers Signed-off-by: jademcosta <jade.costa@nubank.com.br> * refactor: move validPrefix to prefixed_bucket logic Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * Enhance the documentation about prefix. Signed-off-by: jademcosta <jademcosta@gmail.com> * Fix format Signed-off-by: jademcosta <jademcosta@gmail.com> * Add prefix entry on bucket config example Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Removing redundancies on prefix checks and tests We already check if the prefix if not empty when creating the bucket. Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Remove redundant YAML unmarshal Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Remove unused parameter Signed-off-by: jademcosta <jade.costa@nubank.com.br> * Remove docs that should be auto-geneated Signed-off-by: jademcosta <jade.costa@nubank.com.br> * refactor: move prefix to config root level Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * add auto generated docs Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> * fix changelog Signed-off-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br> Co-authored-by: Maria Eduarda Duarte <dudammduarte@yahoo.com.br>
- Loading branch information
1 parent
296c4ab
commit 5da60e0
Showing
10 changed files
with
217 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -56,6 +56,7 @@ type: GCS | |
| config: | ||
| bucket: "" | ||
| service_account: "" | ||
| prefix: "" | ||
| ``` | ||
|
|
||
| ## Upload compacted blocks | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| // Copyright (c) The Thanos Authors. | ||
| // Licensed under the Apache License 2.0. | ||
|
|
||
| package objstore | ||
|
|
||
| import ( | ||
| "context" | ||
| "io" | ||
| "strings" | ||
| ) | ||
|
|
||
| type PrefixedBucket struct { | ||
| bkt Bucket | ||
| prefix string | ||
| } | ||
|
|
||
| func NewPrefixedBucket(bkt Bucket, prefix string) Bucket { | ||
| if validPrefix(prefix) { | ||
| return &PrefixedBucket{bkt: bkt, prefix: strings.Trim(prefix, DirDelim)} | ||
| } | ||
|
|
||
| return bkt | ||
| } | ||
|
|
||
| func validPrefix(prefix string) bool { | ||
| prefix = strings.Replace(prefix, "/", "", -1) | ||
| return len(prefix) > 0 | ||
| } | ||
|
|
||
| func conditionalPrefix(prefix, name string) string { | ||
| if len(name) > 0 { | ||
| return withPrefix(prefix, name) | ||
| } | ||
|
|
||
| return name | ||
| } | ||
|
|
||
| func withPrefix(prefix, name string) string { | ||
| return prefix + DirDelim + name | ||
| } | ||
|
|
||
| func (p *PrefixedBucket) Close() error { | ||
| return p.bkt.Close() | ||
| } | ||
|
|
||
| // Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full | ||
| // object name including the prefix of the inspected directory. | ||
| // Entries are passed to function in sorted order. | ||
| func (p *PrefixedBucket) Iter(ctx context.Context, dir string, f func(string) error, options ...IterOption) error { | ||
| pdir := withPrefix(p.prefix, dir) | ||
|
|
||
| return p.bkt.Iter(ctx, pdir, func(s string) error { | ||
| return f(strings.TrimPrefix(s, p.prefix+DirDelim)) | ||
| }, options...) | ||
| } | ||
|
|
||
| // Get returns a reader for the given object name. | ||
| func (p *PrefixedBucket) Get(ctx context.Context, name string) (io.ReadCloser, error) { | ||
| return p.bkt.Get(ctx, conditionalPrefix(p.prefix, name)) | ||
| } | ||
|
|
||
| // GetRange returns a new range reader for the given object name and range. | ||
| func (p *PrefixedBucket) GetRange(ctx context.Context, name string, off int64, length int64) (io.ReadCloser, error) { | ||
| return p.bkt.GetRange(ctx, conditionalPrefix(p.prefix, name), off, length) | ||
| } | ||
|
|
||
| // Exists checks if the given object exists in the bucket. | ||
| func (p *PrefixedBucket) Exists(ctx context.Context, name string) (bool, error) { | ||
| return p.bkt.Exists(ctx, conditionalPrefix(p.prefix, name)) | ||
| } | ||
|
|
||
| // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. | ||
| func (p *PrefixedBucket) IsObjNotFoundErr(err error) bool { | ||
| return p.bkt.IsObjNotFoundErr(err) | ||
| } | ||
|
|
||
| // Attributes returns information about the specified object. | ||
| func (p PrefixedBucket) Attributes(ctx context.Context, name string) (ObjectAttributes, error) { | ||
| return p.bkt.Attributes(ctx, conditionalPrefix(p.prefix, name)) | ||
| } | ||
|
|
||
| // Upload the contents of the reader as an object into the bucket. | ||
| // Upload should be idempotent. | ||
| func (p *PrefixedBucket) Upload(ctx context.Context, name string, r io.Reader) error { | ||
| return p.bkt.Upload(ctx, conditionalPrefix(p.prefix, name), r) | ||
| } | ||
|
|
||
| // Delete removes the object with the given name. | ||
| // If object does not exists in the moment of deletion, Delete should throw error. | ||
| func (p *PrefixedBucket) Delete(ctx context.Context, name string) error { | ||
| return p.bkt.Delete(ctx, conditionalPrefix(p.prefix, name)) | ||
| } | ||
|
|
||
| // Name returns the bucket name for the provider. | ||
| func (p *PrefixedBucket) Name() string { | ||
| return p.bkt.Name() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| // Copyright (c) The Thanos Authors. | ||
| // Licensed under the Apache License 2.0. | ||
|
|
||
| package objstore | ||
|
|
||
| import ( | ||
| "context" | ||
| "io/ioutil" | ||
| "sort" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/thanos-io/thanos/pkg/testutil" | ||
| ) | ||
|
|
||
| func TestPrefixedBucket_Acceptance(t *testing.T) { | ||
|
|
||
| prefixes := []string{ | ||
| "/someprefix/anotherprefix/", | ||
| "someprefix/anotherprefix/", | ||
| "someprefix/anotherprefix", | ||
| "someprefix/", | ||
| "someprefix"} | ||
|
|
||
| for _, prefix := range prefixes { | ||
| AcceptanceTest(t, NewPrefixedBucket(NewInMemBucket(), prefix)) | ||
| UsesPrefixTest(t, NewInMemBucket(), prefix) | ||
| } | ||
| } | ||
|
|
||
| func UsesPrefixTest(t *testing.T, bkt Bucket, prefix string) { | ||
| testutil.Ok(t, bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/file1.jpg", strings.NewReader("test-data1"))) | ||
|
|
||
| pBkt := NewPrefixedBucket(bkt, prefix) | ||
| rc1, err := pBkt.Get(context.Background(), "file1.jpg") | ||
| testutil.Ok(t, err) | ||
|
|
||
| testutil.Ok(t, err) | ||
| defer func() { testutil.Ok(t, rc1.Close()) }() | ||
| content, err := ioutil.ReadAll(rc1) | ||
| testutil.Ok(t, err) | ||
| testutil.Equals(t, "test-data1", string(content)) | ||
|
|
||
| testutil.Ok(t, pBkt.Upload(context.Background(), "file2.jpg", strings.NewReader("test-data2"))) | ||
| rc2, err := bkt.Get(context.Background(), strings.Trim(prefix, "/")+"/file2.jpg") | ||
| testutil.Ok(t, err) | ||
| defer func() { testutil.Ok(t, rc2.Close()) }() | ||
| contentUpload, err := ioutil.ReadAll(rc2) | ||
| testutil.Ok(t, err) | ||
| testutil.Equals(t, "test-data2", string(contentUpload)) | ||
|
|
||
| testutil.Ok(t, pBkt.Delete(context.Background(), "file2.jpg")) | ||
| _, err = bkt.Get(context.Background(), strings.Trim(prefix, "/")+"/file2.jpg") | ||
| testutil.NotOk(t, err) | ||
| testutil.Assert(t, pBkt.IsObjNotFoundErr(err), "expected not found error got %s", err) | ||
|
|
||
| rc3, err := pBkt.GetRange(context.Background(), "file1.jpg", 1, 3) | ||
| testutil.Ok(t, err) | ||
| defer func() { testutil.Ok(t, rc3.Close()) }() | ||
| content, err = ioutil.ReadAll(rc3) | ||
| testutil.Ok(t, err) | ||
| testutil.Equals(t, "est", string(content)) | ||
|
|
||
| ok, err := pBkt.Exists(context.Background(), "file1.jpg") | ||
| testutil.Ok(t, err) | ||
| testutil.Assert(t, ok, "expected exits") | ||
|
|
||
| attrs, err := pBkt.Attributes(context.Background(), "file1.jpg") | ||
| testutil.Ok(t, err) | ||
| testutil.Assert(t, attrs.Size == 10, "expected size to be equal to 10") | ||
|
|
||
| testutil.Ok(t, bkt.Upload(context.Background(), strings.Trim(prefix, "/")+"/dir/file1.jpg", strings.NewReader("test-data1"))) | ||
| seen := []string{} | ||
| testutil.Ok(t, pBkt.Iter(context.Background(), "", func(fn string) error { | ||
| seen = append(seen, fn) | ||
| return nil | ||
| }, WithRecursiveIter)) | ||
| expected := []string{"dir/file1.jpg", "file1.jpg"} | ||
| sort.Strings(expected) | ||
| sort.Strings(seen) | ||
| testutil.Equals(t, expected, seen) | ||
|
|
||
| seen = []string{} | ||
| testutil.Ok(t, pBkt.Iter(context.Background(), "", func(fn string) error { | ||
| seen = append(seen, fn) | ||
| return nil | ||
| })) | ||
| expected = []string{"dir/", "file1.jpg"} | ||
| sort.Strings(expected) | ||
| sort.Strings(seen) | ||
| testutil.Equals(t, expected, seen) | ||
| } |