forked from grafana/loki
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Storage: Allow setting a constant prefix for all created keys (grafan…
…a#10096) **What this PR does / why we need it**: Adds a new option under the aws stanza named key_prefix. This is useful if you have many Loki installations and do not want to create a million buckets. This is different than `compactor.shared_store_key_prefix` because it also affects eg the `loki_cluster_seed.json` file. **Which issue(s) this PR fixes**: Fixes grafana#5889 **Special notes for your reviewer**: **Checklist** - [x] Reviewed the [`CONTRIBUTING.md`](https://github.com/grafana/loki/blob/main/CONTRIBUTING.md) guide (**required**) - [x] Documentation added - [x] Tests updated - [x] `CHANGELOG.md` updated - [ ] If the change is worth mentioning in the release notes, add `add-to-release-notes` label - [x] Changes that require user attention or interaction to upgrade are documented in `docs/sources/setup/upgrade/_index.md` - [ ] For Helm chart changes bump the Helm chart version in `production/helm/loki/Chart.yaml` and update `production/helm/loki/CHANGELOG.md` and `production/helm/loki/README.md`. [Example PR](grafana@d10549e) --------- Signed-off-by: Edward Welch <edward.welch@grafana.com> Co-authored-by: Ed Welch <edward.welch@grafana.com>
- Loading branch information
Showing
9 changed files
with
180 additions
and
74 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
package client | ||
|
||
import ( | ||
"context" | ||
"io" | ||
"strings" | ||
) | ||
|
||
type PrefixedObjectClient struct { | ||
downstreamClient ObjectClient | ||
prefix string | ||
} | ||
|
||
func NewPrefixedObjectClient(downstreamClient ObjectClient, prefix string) ObjectClient { | ||
return PrefixedObjectClient{downstreamClient: downstreamClient, prefix: prefix} | ||
} | ||
|
||
func (p PrefixedObjectClient) PutObject(ctx context.Context, objectKey string, object io.ReadSeeker) error { | ||
return p.downstreamClient.PutObject(ctx, p.prefix+objectKey, object) | ||
} | ||
|
||
func (p PrefixedObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { | ||
return p.downstreamClient.ObjectExists(ctx, p.prefix+objectKey) | ||
} | ||
|
||
func (p PrefixedObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error) { | ||
return p.downstreamClient.GetObject(ctx, p.prefix+objectKey) | ||
} | ||
|
||
func (p PrefixedObjectClient) List(ctx context.Context, prefix, delimiter string) ([]StorageObject, []StorageCommonPrefix, error) { | ||
objects, commonPrefixes, err := p.downstreamClient.List(ctx, p.prefix+prefix, delimiter) | ||
if err != nil { | ||
return nil, nil, err | ||
} | ||
|
||
for i := range objects { | ||
objects[i].Key = strings.TrimPrefix(objects[i].Key, p.prefix) | ||
} | ||
|
||
for i := range commonPrefixes { | ||
commonPrefixes[i] = StorageCommonPrefix(strings.TrimPrefix(string(commonPrefixes[i]), p.prefix)) | ||
} | ||
|
||
return objects, commonPrefixes, nil | ||
} | ||
|
||
func (p PrefixedObjectClient) DeleteObject(ctx context.Context, objectKey string) error { | ||
return p.downstreamClient.DeleteObject(ctx, p.prefix+objectKey) | ||
} | ||
|
||
func (p PrefixedObjectClient) IsObjectNotFoundErr(err error) bool { | ||
return p.downstreamClient.IsObjectNotFoundErr(err) | ||
} | ||
|
||
func (p PrefixedObjectClient) IsRetryableErr(err error) bool { | ||
return p.downstreamClient.IsRetryableErr(err) | ||
} | ||
|
||
func (p PrefixedObjectClient) Stop() { | ||
p.downstreamClient.Stop() | ||
} | ||
|
||
func (p PrefixedObjectClient) GetDownstream() ObjectClient { | ||
return p.downstreamClient | ||
} | ||
|
||
func (p PrefixedObjectClient) GetPrefix() string { | ||
return p.prefix | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
63 changes: 0 additions & 63 deletions
63
pkg/storage/stores/shipper/indexshipper/storage/prefixed_object_client.go
This file was deleted.
Oops, something went wrong.