Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(storage): add Autoclass support #6828

Merged
merged 13 commits into from Nov 1, 2022
79 changes: 79 additions & 0 deletions storage/bucket.go
Expand Up @@ -444,6 +444,11 @@ type BucketAttrs struct {
// See https://cloud.google.com/storage/docs/managing-turbo-replication for
// more information.
RPO RPO

// Autoclass holds the bucket's autoclass configuration. If enabled,
// allows for the automatic selection of the best storage class
// based on object access patterns.
Autoclass *Autoclass
}

// BucketPolicyOnly is an alias for UniformBucketLevelAccess.
Expand Down Expand Up @@ -710,6 +715,20 @@ type CustomPlacementConfig struct {
DataLocations []string
}

// Autoclass holds the bucket's autoclass configuration. If enabled,
// allows for the automatic selection of the best storage class
// based on object access patterns. See
// https://cloud.google.com/storage/docs/using-autoclass for more information.
type Autoclass struct {
// Enabled specifies whether the autoclass feature is enabled
// on the bucket.
Enabled bool
// ToggleTime is the time from which Autoclass was last toggled.
// If Autoclass is enabled when the bucket is created, the ToggleTime
// is set to the bucket creation time. This field is read-only.
ToggleTime time.Time
}

func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
if b == nil {
return nil, nil
Expand Down Expand Up @@ -744,6 +763,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
ProjectNumber: b.ProjectNumber,
RPO: toRPO(b),
CustomPlacementConfig: customPlacementFromRaw(b.CustomPlacementConfig),
Autoclass: toAutoclass(b.Autoclass),
}, nil
}

Expand Down Expand Up @@ -776,6 +796,7 @@ func newBucketFromProto(b *storagepb.Bucket) *BucketAttrs {
RPO: toRPOFromProto(b),
CustomPlacementConfig: customPlacementFromProto(b.GetCustomPlacementConfig()),
ProjectNumber: parseProjectNumber(b.GetProject()), // this can return 0 the project resource name is ID based
Autoclass: toAutoclassFromProto(b.GetAutoclass()),
}
}

Expand Down Expand Up @@ -830,6 +851,7 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket {
IamConfiguration: bktIAM,
Rpo: b.RPO.String(),
CustomPlacementConfig: b.CustomPlacementConfig.toRawCustomPlacement(),
Autoclass: b.Autoclass.toRawBucketAutoclass(),
}
}

Expand Down Expand Up @@ -889,6 +911,7 @@ func (b *BucketAttrs) toProtoBucket() *storagepb.Bucket {
IamConfig: bktIAM,
Rpo: b.RPO.String(),
CustomPlacementConfig: b.CustomPlacementConfig.toProtoCustomPlacement(),
Autoclass: b.Autoclass.toProtoBucketAutoclass(),
}
}

Expand Down Expand Up @@ -964,6 +987,7 @@ func (ua *BucketAttrsToUpdate) toProtoBucket() *storagepb.Bucket {
Website: ua.Website.toProtoBucketWebsite(),
IamConfig: bktIAM,
Rpo: ua.RPO.String(),
Autoclass: ua.Autoclass.toProtoBucketAutoclass(),
}
}

Expand Down Expand Up @@ -1079,6 +1103,10 @@ type BucketAttrsToUpdate struct {
// more information.
RPO RPO

// If set, updates the autoclass configuration of the bucket.
// See <TBD> for more information.
cojenco marked this conversation as resolved.
Show resolved Hide resolved
Autoclass *Autoclass

// acl is the list of access control rules on the bucket.
// It is unexported and only used internally by the gRPC client.
// Library users should use ACLHandle methods directly.
Expand Down Expand Up @@ -1192,6 +1220,11 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
rb.Website = ua.Website.toRawBucketWebsite()
}
}
if ua.Autoclass != nil {
rb.Autoclass = &raw.BucketAutoclass{
Enabled: ua.Autoclass.Enabled,
}
}
if ua.PredefinedACL != "" {
// Clear ACL or the call will fail.
rb.Acl = nil
Expand Down Expand Up @@ -1892,6 +1925,52 @@ func customPlacementFromProto(c *storagepb.Bucket_CustomPlacementConfig) *Custom
return &CustomPlacementConfig{DataLocations: c.GetDataLocations()}
}

func (a *Autoclass) toRawBucketAutoclass() *raw.BucketAutoclass {
cojenco marked this conversation as resolved.
Show resolved Hide resolved
if a == nil {
return nil
}
// Excluding read only field ToggleTime.
return &raw.BucketAutoclass{
Enabled: a.Enabled,
}
}

func (a *Autoclass) toProtoBucketAutoclass() *storagepb.Bucket_Autoclass {
if a == nil {
return nil
}
// Excluding read only field ToggleTime.
return &storagepb.Bucket_Autoclass{
Enabled: a.Enabled,
}
}

func toAutoclass(a *raw.BucketAutoclass) *Autoclass {
cojenco marked this conversation as resolved.
Show resolved Hide resolved
if a == nil {
return nil
}
t, err := time.Parse(time.RFC3339, a.ToggleTime)
if err != nil {
cojenco marked this conversation as resolved.
Show resolved Hide resolved
return &Autoclass{
Enabled: a.Enabled,
}
}
return &Autoclass{
Enabled: a.Enabled,
ToggleTime: t,
}
}

func toAutoclassFromProto(a *storagepb.Bucket_Autoclass) *Autoclass {
if a == nil {
return nil
}
return &Autoclass{
Enabled: a.GetEnabled(),
ToggleTime: a.GetToggleTime().AsTime(),
}
}

// Objects returns an iterator over the objects in the bucket that match the
// Query q. If q is nil, no filtering is done. Objects will be iterated over
// lexicographically by name.
Expand Down
12 changes: 12 additions & 0 deletions storage/bucket_test.go
Expand Up @@ -60,6 +60,7 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's make sure we have test coverage for the proto converters as well. Also, you should make an integration test to cover the roundtrip with the service (both create and update).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep was meaning to write an integration test... also added 2 tests for the proto converters. @tritone PTAL, thanks!

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for adding these additional tests; didn't realize we were missing these unit tests for proto converter methods. We should consider auditing and filling others in if they are missing. @noahdietz @BrennaEpp

Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
Autoclass: &Autoclass{Enabled: true},
Lifecycle: Lifecycle{
Rules: []LifecycleRule{{
Action: LifecycleAction{
Expand Down Expand Up @@ -163,6 +164,7 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
Encryption: &raw.BucketEncryption{DefaultKmsKeyName: "key"},
Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
Autoclass: &raw.BucketAutoclass{Enabled: true},
Lifecycle: &raw.BucketLifecycle{
Rule: []*raw.BucketLifecycleRule{{
Action: &raw.BucketLifecycleRuleAction{
Expand Down Expand Up @@ -391,6 +393,7 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
StorageClass: "NEARLINE",
Autoclass: &Autoclass{Enabled: false},
}
au.SetLabel("a", "foo")
au.DeleteLabel("b")
Expand Down Expand Up @@ -434,6 +437,7 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
StorageClass: "NEARLINE",
Autoclass: &raw.BucketAutoclass{Enabled: false},
ForceSendFields: []string{"DefaultEventBasedHold", "Lifecycle"},
}
if msg := testutil.Diff(got, want); msg != "" {
Expand Down Expand Up @@ -638,6 +642,10 @@ func TestNewBucket(t *testing.T) {
Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
ProjectNumber: 123231313,
Autoclass: &raw.BucketAutoclass{
Enabled: true,
ToggleTime: "2017-10-23T04:05:06Z",
},
}
want := &BucketAttrs{
Name: "name",
Expand Down Expand Up @@ -688,6 +696,10 @@ func TestNewBucket(t *testing.T) {
DefaultObjectACL: nil,
LocationType: "dual-region",
ProjectNumber: 123231313,
Autoclass: &Autoclass{
Enabled: true,
ToggleTime: time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
},
}
got, err := newBucket(rb)
if err != nil {
Expand Down