Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(bigtable): add GC policy to FamilyInfo. #6234

Merged
merged 41 commits into from Jun 22, 2022
Merged
Show file tree
Hide file tree
Changes from 40 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
dc6e491
Add GC policy proto to FamilyInfo.
hoangpham95 May 4, 2022
736e1bf
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 May 4, 2022
265d7f6
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 May 5, 2022
1861f7f
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 May 9, 2022
992dbff
Add PolicyType and expose GCPolicyObject.
hoangpham95 May 9, 2022
dc64bdd
Add PolicyType alias to int.
hoangpham95 May 9, 2022
5741a5a
Add new TypedGCPolicy to prevent breaking changes
hoangpham95 May 12, 2022
f7a68fc
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 May 12, 2022
a215371
Remove unused methods
hoangpham95 May 12, 2022
ee17e07
Address PR comments.
hoangpham95 May 16, 2022
c2eff7c
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 May 16, 2022
f93358d
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion May 16, 2022
3b2c864
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 2, 2022
6597b60
Change variable name to Unspecified instead of Unknown.
hoangpham95 Jun 2, 2022
bace825
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 2, 2022
61dcb35
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 4, 2022
696172a
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 7, 2022
8eb8f75
Fix go fmt.
hoangpham95 Jun 7, 2022
f3df3da
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 7, 2022
330fecb
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 8, 2022
8c1b978
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 8, 2022
71fb066
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 9, 2022
0e2a815
Address build failure.
hoangpham95 Jun 10, 2022
0d12c0a
Add FullGCPolicy to admin.go.
hoangpham95 Jun 13, 2022
d10c1cf
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 13, 2022
674f607
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 13, 2022
83108bc
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 13, 2022
778031f
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 14, 2022
d30b02f
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 14, 2022
b19cdbc
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 16, 2022
cf1fae8
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 16, 2022
b2f7785
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 16, 2022
de859e9
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 16, 2022
849e7fb
Merge branch 'main' into hoangpham95/add-gc-proto
telpirion Jun 17, 2022
444f66b
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 20, 2022
e7f871d
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 21, 2022
149d191
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 21, 2022
4b9651b
Merge branch 'main' into hoangpham95/add-gc-proto
hoangpham95 Jun 21, 2022
fc286b6
Merge branch 'main' into add-gc-policy-family-info
hoangpham95 Jun 22, 2022
55315a5
Merge branch 'main' into add-gc-policy-family-info
telpirion Jun 22, 2022
fcd5d53
Merge branch 'main' into add-gc-policy-family-info
hoangpham95 Jun 22, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 8 additions & 3 deletions bigtable/admin.go
Expand Up @@ -310,8 +310,9 @@ type TableInfo struct {

// FamilyInfo represents information about a column family.
type FamilyInfo struct {
Name string
GCPolicy string
Name string
GCPolicy string
FullGCPolicy GCPolicy
}

func (ac *AdminClient) getTable(ctx context.Context, table string, view btapb.Table_View) (*btapb.Table, error) {
Expand Down Expand Up @@ -347,7 +348,11 @@ func (ac *AdminClient) TableInfo(ctx context.Context, table string) (*TableInfo,
ti := &TableInfo{}
for name, fam := range res.ColumnFamilies {
ti.Families = append(ti.Families, name)
ti.FamilyInfos = append(ti.FamilyInfos, FamilyInfo{Name: name, GCPolicy: GCRuleToString(fam.GcRule)})
ti.FamilyInfos = append(ti.FamilyInfos, FamilyInfo{
Name: name,
GCPolicy: GCRuleToString(fam.GcRule),
FullGCPolicy: gcRuleToPolicy(fam.GcRule),
})
}
return ti, nil
}
Expand Down
44 changes: 44 additions & 0 deletions bigtable/gc.go
Expand Up @@ -25,6 +25,13 @@ import (
bttdpb "google.golang.org/genproto/googleapis/bigtable/admin/v2"
)

type policyType int

const (
policyTypeUnion policyType = iota
policyTypeIntersection
)

// A GCPolicy represents a rule that determines which cells are eligible for garbage collection.
type GCPolicy interface {
String() string
Expand Down Expand Up @@ -158,10 +165,47 @@ func GCRuleToString(rule *bttdpb.GcRule) string {
}
}

func gcRuleToPolicy(rule *bttdpb.GcRule) GCPolicy {
if rule == nil {
return NoGcPolicy()
}
switch r := rule.Rule.(type) {
case *bttdpb.GcRule_Intersection_:
return compoundRuleToPolicy(r.Intersection.Rules, policyTypeIntersection)
case *bttdpb.GcRule_Union_:
return compoundRuleToPolicy(r.Union.Rules, policyTypeUnion)
case *bttdpb.GcRule_MaxAge:
return MaxAgePolicy(time.Duration(r.MaxAge.Seconds) * time.Second)
case *bttdpb.GcRule_MaxNumVersions:
return MaxVersionsPolicy(int(r.MaxNumVersions))
default:
return NoGcPolicy()
}
}

func joinRules(rules []*bttdpb.GcRule, sep string) string {
var chunks []string
for _, r := range rules {
chunks = append(chunks, GCRuleToString(r))
}
return "(" + strings.Join(chunks, sep) + ")"
}

func compoundRuleToPolicy(rules []*bttdpb.GcRule, mode policyType) GCPolicy {
sub := []GCPolicy{}
for _, r := range rules {
p := gcRuleToPolicy(r)
if p.String() != "" {
sub = append(sub, gcRuleToPolicy(r))
}
}

switch mode {
case policyTypeUnion:
return unionPolicy{sub: sub}
case policyTypeIntersection:
return intersectionPolicy{sub: sub}
default:
return NoGcPolicy()
}
}
23 changes: 23 additions & 0 deletions bigtable/gc_test.go
Expand Up @@ -44,3 +44,26 @@ func TestGcRuleToString(t *testing.T) {
}
}
}

func TestGCRuleToPolicy(t *testing.T) {
var tc = []struct {
proto *bttdpb.GcRule
want string
}{
{MaxAgePolicy(72 * time.Hour).proto(), "age() > 3d"},
{MaxVersionsPolicy(5).proto(), "versions() > 5"},
{IntersectionPolicy(MaxAgePolicy(5*time.Hour), MaxVersionsPolicy(4)).proto(),
"(age() > 5h && versions() > 4)"},
{UnionPolicy(MaxAgePolicy(5*time.Hour), MaxVersionsPolicy(4)).proto(),
"(age() > 5h || versions() > 4)"},
{IntersectionPolicy(UnionPolicy(MaxAgePolicy(5*time.Hour), MaxVersionsPolicy(4)), MaxVersionsPolicy(8)).proto(),
"((age() > 5h || versions() > 4) && versions() > 8)"},
}

for _, test := range tc {
got := gcRuleToPolicy(test.proto)
if got.String() != test.want {
t.Errorf("got gc rule %v, want: %v", got, test.want)
}
}
}