Skip to content

Commit

Permalink
blob/s3blob: adds support for per-request s3v2 Options, implementing #…
Browse files Browse the repository at this point in the history 
…3214 (#3238)
  • Loading branch information
@dkolbly
dkolbly committed Apr 4, 2023
1 parent aeb26ed commit 04a775a
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 29 deletions.
60 changes: 38 additions & 22 deletions blob/s3blob/s3blob.go
View file
Expand Up @@ -44,12 +44,12 @@
// s3blob exposes the following types for As:
// - Bucket: (V1) *s3.S3; (V2) *s3v2.Client
// - Error: (V1) awserr.Error; (V2) any error type returned by the service, notably smithy.APIError
// - ListObject: (V1) s3.Object for objects, s3.CommonPrefix for "directories"; (V2) typesv2.Object for objects, typesv2.CommonPrefix for "directories
// - ListOptions.BeforeList: (V1) *s3.ListObjectsV2Input, or *s3.ListObjectsInput
// when Options.UseLegacyList == true; (V2) *s3v2.ListObjectsV2Input, or *s3v2.ListObjectsInput
// - ListObject: (V1) s3.Object for objects, s3.CommonPrefix for "directories"; (V2) typesv2.Object for objects, typesv2.CommonPrefix for "directories"
// - ListOptions.BeforeList: (V1) *s3.ListObjectsV2Input or *s3.ListObjectsInput
// when Options.UseLegacyList == true; (V2) *s3v2.ListObjectsV2Input or *[]func(*s3v2.Options), or *s3v2.ListObjectsInput
// when Options.UseLegacyList == true
// - Reader: (V1) s3.GetObjectOutput; (V2) s3v2.GetObjectInput
// - ReaderOptions.BeforeRead: (V1) *s3.GetObjectInput; (V2) *s3v2.GetObjectInput
// - ReaderOptions.BeforeRead: (V1) *s3.GetObjectInput; (V2) *s3v2.GetObjectInput or *[]func(*s3v2.Options)
// - Attributes: (V1) s3.HeadObjectOutput; (V2)s3v2.HeadObjectOutput
// - CopyOptions.BeforeCopy: *(V1) s3.CopyObjectInput; (V2) s3v2.CopyObjectInput
// - WriterOptions.BeforeWrite: (V1) *s3manager.UploadInput, *s3manager.Uploader; (V2) *s3v2.PutObjectInput, *s3v2manager.Uploader
Expand Down Expand Up @@ -528,20 +528,24 @@ func (b *bucket) ListPaged(ctx context.Context, opts *driver.ListOptions) (*driv

func (b *bucket) listObjectsV2(ctx context.Context, in *s3v2.ListObjectsV2Input, opts *driver.ListOptions) (*s3v2.ListObjectsV2Output, error) {
if !b.useLegacyList {
var varopt []func(*s3v2.Options)
if opts.BeforeList != nil {
asFunc := func(i interface{}) bool {
p, ok := i.(**s3v2.ListObjectsV2Input)
if !ok {
return false
if p, ok := i.(**s3v2.ListObjectsV2Input); ok {
*p = in
return true
}
*p = in
return true
if p, ok := i.(**[]func(*s3v2.Options)); ok {
*p = &varopt
return true
}
return false
}
if err := opts.BeforeList(asFunc); err != nil {
return nil, err
}
}
return b.clientV2.ListObjectsV2(ctx, in)
return b.clientV2.ListObjectsV2(ctx, in, varopt...)
}

// Use the legacy ListObjects request.
Expand Down Expand Up @@ -589,12 +593,11 @@ func (b *bucket) listObjects(ctx context.Context, in *s3.ListObjectsV2Input, opt
if !b.useLegacyList {
if opts.BeforeList != nil {
asFunc := func(i interface{}) bool {
p, ok := i.(**s3.ListObjectsV2Input)
if !ok {
return false
if p, ok := i.(**s3.ListObjectsV2Input); ok {
*p = in
return true
}
*p = in
return true
return false
}
if err := opts.BeforeList(asFunc); err != nil {
return nil, err
Expand Down Expand Up @@ -776,19 +779,24 @@ func (b *bucket) NewRangeReader(ctx context.Context, key string, offset, length
Key: aws.String(key),
Range: byteRange,
}
var varopt []func(*s3v2.Options)
if opts.BeforeRead != nil {
asFunc := func(i interface{}) bool {
if p, ok := i.(**s3v2.GetObjectInput); ok {
*p = in
return true
}
if p, ok := i.(**[]func(*s3v2.Options)); ok {
*p = &varopt
return true
}
return false
}
if err := opts.BeforeRead(asFunc); err != nil {
return nil, err
}
}
resp, err := b.clientV2.GetObject(ctx, in)
resp, err := b.clientV2.GetObject(ctx, in, varopt...)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -960,14 +968,22 @@ func (b *bucket) NewTypedWriter(ctx context.Context, key string, contentType str
}
if opts.BeforeWrite != nil {
asFunc := func(i interface{}) bool {
pu, ok := i.(**s3managerv2.Uploader)
if ok {
*pu = uploaderV2
// Note that since the Go CDK Blob
// abstraction does not expose AWS's
// Uploader concept, there does not
// appear to be any utility in
// exposing the options list to the v2
// Uploader's Upload() method.
// Instead, applications can
// manipulate the exposed *Uploader
// directly, including by setting
// ClientOptions if needed.
if p, ok := i.(**s3managerv2.Uploader); ok {
*p = uploaderV2
return true
}
pui, ok := i.(**s3v2.PutObjectInput)
if ok {
*pui = reqV2
if p, ok := i.(**s3v2.PutObjectInput); ok {
*p = reqV2
return true
}
return false
Expand Down
24 changes: 17 additions & 7 deletions blob/s3blob/s3blob_test.go
View file
Expand Up @@ -129,7 +129,7 @@ func BenchmarkS3blob(b *testing.B) {

const language = "nl"

// verifyContentLanguage uses As to access the underlying GCS types and
// verifyContentLanguage uses As to access the underlying AWS types and
// read/write the ContentLanguage field.
type verifyContentLanguage struct {
useV2 bool
Expand Down Expand Up @@ -172,8 +172,11 @@ func (v verifyContentLanguage) ErrorCheck(b *blob.Bucket, err error) error {

func (v verifyContentLanguage) BeforeRead(as func(interface{}) bool) error {
if v.useV2 {
var req *s3v2.GetObjectInput
if !as(&req) {
var (
req *s3v2.GetObjectInput
opts *[]func(*s3v2.Options)
)
if !as(&req) || !as(&opts) {
return errors.New("BeforeRead As failed")
}
return nil
Expand All @@ -187,8 +190,11 @@ func (v verifyContentLanguage) BeforeRead(as func(interface{}) bool) error {

func (v verifyContentLanguage) BeforeWrite(as func(interface{}) bool) error {
if v.useV2 {
var req *s3v2.PutObjectInput
if !as(&req) {
var (
req *s3v2.PutObjectInput
uploader *s3managerv2.Uploader
)
if !as(&req) || !as(&uploader) {
return errors.New("Writer.As failed for PutObjectInput")
}
req.ContentLanguage = aws.String(language)
Expand Down Expand Up @@ -233,10 +239,14 @@ func (v verifyContentLanguage) BeforeList(as func(interface{}) bool) error {
return errors.New("List.As failed")
}
} else {
var req *s3v2.ListObjectsV2Input
if !as(&req) {
var (
list *s3v2.ListObjectsV2Input
opts *[]func(*s3v2.Options)
)
if !as(&list) || !as(&opts) {
return errors.New("List.As failed")
}
return nil
}
return nil
}
Expand Down

0 comments on commit 04a775a

Please sign in to comment.