forked from thanos-io/thanos
-
Notifications
You must be signed in to change notification settings - Fork 0
/
markers.go
122 lines (103 loc) · 4.81 KB
/
markers.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
// Copyright (c) The Thanos Authors.
// Licensed under the Apache License 2.0.
package metadata
import (
"context"
"encoding/json"
"io/ioutil"
"path"
"github.com/go-kit/log"
"github.com/oklog/ulid"
"github.com/pkg/errors"
"github.com/thanos-io/objstore"
"github.com/thanos-io/thanos/pkg/runutil"
)
const (
// DeletionMarkFilename is the known json filename for optional file storing details about when block is marked for deletion.
// If such file is present in block dir, it means the block is meant to be deleted after certain delay.
DeletionMarkFilename = "deletion-mark.json"
// NoCompactMarkFilename is the known json filename for optional file storing details about why block has to be excluded from compaction.
// If such file is present in block dir, it means the block has to excluded from compaction (both vertical and horizontal) or rewrite (e.g deletions).
NoCompactMarkFilename = "no-compact-mark.json"
// DeletionMarkVersion1 is the version of deletion-mark file supported by Thanos.
DeletionMarkVersion1 = 1
// NoCompactMarkVersion1 is the version of no-compact-mark file supported by Thanos.
NoCompactMarkVersion1 = 1
)
var (
// ErrorMarkerNotFound is the error when marker file is not found.
ErrorMarkerNotFound = errors.New("marker not found")
// ErrorUnmarshalMarker is the error when unmarshalling marker JSON file.
// This error can occur because marker has been partially uploaded to block storage
// or the marker file is not a valid json file.
ErrorUnmarshalMarker = errors.New("unmarshal marker JSON")
)
type Marker interface {
markerFilename() string
}
// DeletionMark stores block id and when block was marked for deletion.
type DeletionMark struct {
// ID of the tsdb block.
ID ulid.ULID `json:"id"`
// Version of the file.
Version int `json:"version"`
// Details is a human readable string giving details of reason.
Details string `json:"details,omitempty"`
// DeletionTime is a unix timestamp of when the block was marked to be deleted.
DeletionTime int64 `json:"deletion_time"`
}
func (m *DeletionMark) markerFilename() string { return DeletionMarkFilename }
// NoCompactReason is a reason for a block to be excluded from compaction.
type NoCompactReason string
const (
// ManualNoCompactReason is a custom reason of excluding from compaction that should be added when no-compact mark is added for unknown/user specified reason.
ManualNoCompactReason NoCompactReason = "manual"
// IndexSizeExceedingNoCompactReason is a reason of index being too big (for example exceeding 64GB limit: https://github.com/thanos-io/thanos/issues/1424)
// This reason can be ignored when vertical block sharding will be implemented.
IndexSizeExceedingNoCompactReason = "index-size-exceeding"
// OutOfOrderChunksNoCompactReason is a reason of to no compact block with index contains out of order chunk so that the compaction is not blocked.
OutOfOrderChunksNoCompactReason = "block-index-out-of-order-chunk"
)
// NoCompactMark marker stores reason of block being excluded from compaction if needed.
type NoCompactMark struct {
// ID of the tsdb block.
ID ulid.ULID `json:"id"`
// Version of the file.
Version int `json:"version"`
// Details is a human readable string giving details of reason.
Details string `json:"details,omitempty"`
// NoCompactTime is a unix timestamp of when the block was marked for no compact.
NoCompactTime int64 `json:"no_compact_time"`
Reason NoCompactReason `json:"reason"`
}
func (n *NoCompactMark) markerFilename() string { return NoCompactMarkFilename }
// ReadMarker reads the given mark file from <dir>/<marker filename>.json in bucket.
func ReadMarker(ctx context.Context, logger log.Logger, bkt objstore.InstrumentedBucketReader, dir string, marker Marker) error {
markerFile := path.Join(dir, marker.markerFilename())
r, err := bkt.ReaderWithExpectedErrs(bkt.IsObjNotFoundErr).Get(ctx, markerFile)
if err != nil {
if bkt.IsObjNotFoundErr(err) {
return ErrorMarkerNotFound
}
return errors.Wrapf(err, "get file: %s", markerFile)
}
defer runutil.CloseWithLogOnErr(logger, r, "close bkt marker reader")
metaContent, err := ioutil.ReadAll(r)
if err != nil {
return errors.Wrapf(err, "read file: %s", markerFile)
}
if err := json.Unmarshal(metaContent, marker); err != nil {
return errors.Wrapf(ErrorUnmarshalMarker, "file: %s; err: %v", markerFile, err.Error())
}
switch marker.markerFilename() {
case NoCompactMarkFilename:
if version := marker.(*NoCompactMark).Version; version != NoCompactMarkVersion1 {
return errors.Errorf("unexpected no-compact-mark file version %d, expected %d", version, NoCompactMarkVersion1)
}
case DeletionMarkFilename:
if version := marker.(*DeletionMark).Version; version != DeletionMarkVersion1 {
return errors.Errorf("unexpected deletion-mark file version %d, expected %d", version, DeletionMarkVersion1)
}
}
return nil
}