From 165e3dc3a506e9c3f5871a897aa33ac2c5ced99b Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Mon, 27 Dec 2021 18:55:57 +0100 Subject: [PATCH] GCS fs: move all gcsfs related implementations to its own package this way we don't force any application that import afero to include gcfs deps in its binary --- gcsfs/file.go | 2 +- gcsfs/file_info.go | 2 +- gcsfs/file_resource.go | 2 +- gcsfs/fs.go | 48 +++++++++---------- .../gcs-fake-service-account.json | 0 gcs.go => gcsfs/gcs.go | 25 +++++----- gcs_mocks.go => gcsfs/gcs_mocks.go | 37 +++++++------- gcs_test.go => gcsfs/gcs_test.go | 17 ++++--- 8 files changed, 65 insertions(+), 68 deletions(-) rename gcs-fake-service-account.json => gcsfs/gcs-fake-service-account.json (100%) rename gcs.go => gcsfs/gcs.go (88%) rename gcs_mocks.go => gcsfs/gcs_mocks.go (91%) rename gcs_test.go => gcsfs/gcs_test.go (98%) diff --git a/gcsfs/file.go b/gcsfs/file.go index b3b3d892..a3d491ec 100644 --- a/gcsfs/file.go +++ b/gcsfs/file.go @@ -44,7 +44,7 @@ type GcsFile struct { func NewGcsFile( ctx context.Context, - fs *GcsFs, + fs *Fs, obj stiface.ObjectHandle, openFlags int, // Unused: there is no use to the file mode in GCloud just yet - but we keep it here, just in case we need it diff --git a/gcsfs/file_info.go b/gcsfs/file_info.go index 2cccb7ca..0a42d38f 100644 --- a/gcsfs/file_info.go +++ b/gcsfs/file_info.go @@ -37,7 +37,7 @@ type FileInfo struct { fileMode os.FileMode } -func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error) { +func newFileInfo(name string, fs *Fs, fileMode os.FileMode) (*FileInfo, error) { res := &FileInfo{ name: name, size: folderSize, diff --git a/gcsfs/file_resource.go b/gcsfs/file_resource.go index b25d0e86..c25ad51d 100644 --- a/gcsfs/file_resource.go +++ b/gcsfs/file_resource.go @@ -40,7 +40,7 @@ const ( type gcsFileResource struct { ctx context.Context - fs *GcsFs + fs *Fs obj stiface.ObjectHandle name string diff --git a/gcsfs/fs.go b/gcsfs/fs.go index c4d899c7..dc099b1a 100644 --- a/gcsfs/fs.go +++ b/gcsfs/fs.go @@ -33,8 +33,8 @@ const ( gsPrefix = "gs://" ) -// GcsFs is a Fs implementation that uses functions provided by google cloud storage -type GcsFs struct { +// Fs is a Fs implementation that uses functions provided by google cloud storage +type Fs struct { ctx context.Context client stiface.Client separator string @@ -45,12 +45,12 @@ type GcsFs struct { autoRemoveEmptyFolders bool //trigger for creating "virtual folders" (not required by GCSs) } -func NewGcsFs(ctx context.Context, client stiface.Client) *GcsFs { +func NewGcsFs(ctx context.Context, client stiface.Client) *Fs { return NewGcsFsWithSeparator(ctx, client, "/") } -func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *GcsFs { - return &GcsFs{ +func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *Fs { + return &Fs{ ctx: ctx, client: client, separator: folderSep, @@ -61,17 +61,17 @@ func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep } // normSeparators will normalize all "\\" and "/" to the provided separator -func (fs *GcsFs) normSeparators(s string) string { +func (fs *Fs) normSeparators(s string) string { return strings.Replace(strings.Replace(s, "\\", fs.separator, -1), "/", fs.separator, -1) } -func (fs *GcsFs) ensureTrailingSeparator(s string) string { +func (fs *Fs) ensureTrailingSeparator(s string) string { if len(s) > 0 && !strings.HasSuffix(s, fs.separator) { return s + fs.separator } return s } -func (fs *GcsFs) ensureNoLeadingSeparator(s string) string { +func (fs *Fs) ensureNoLeadingSeparator(s string) string { if len(s) > 0 && strings.HasPrefix(s, fs.separator) { s = s[len(fs.separator):] } @@ -94,13 +94,13 @@ func validateName(s string) error { } // Splits provided name into bucket name and path -func (fs *GcsFs) splitName(name string) (bucketName string, path string) { +func (fs *Fs) splitName(name string) (bucketName string, path string) { splitName := strings.Split(name, fs.separator) return splitName[0], strings.Join(splitName[1:], fs.separator) } -func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) { +func (fs *Fs) getBucket(name string) (stiface.BucketHandle, error) { bucket := fs.buckets[name] if bucket == nil { bucket = fs.client.Bucket(name) @@ -112,7 +112,7 @@ func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) { return bucket, nil } -func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) { +func (fs *Fs) getObj(name string) (stiface.ObjectHandle, error) { bucketName, path := fs.splitName(name) bucket, err := fs.getBucket(bucketName) @@ -123,9 +123,9 @@ func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) { return bucket.Object(path), nil } -func (fs *GcsFs) Name() string { return "GcsFs" } +func (fs *Fs) Name() string { return "GcsFs" } -func (fs *GcsFs) Create(name string) (*GcsFile, error) { +func (fs *Fs) Create(name string) (*GcsFile, error) { name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) if err := validateName(name); err != nil { return nil, err @@ -156,7 +156,7 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) { return file, nil } -func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error { +func (fs *Fs) Mkdir(name string, _ os.FileMode) error { name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name)))) if err := validateName(name); err != nil { return err @@ -179,7 +179,7 @@ func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error { return w.Close() } -func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error { +func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path)))) if err := validateName(path); err != nil { return err @@ -216,11 +216,11 @@ func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error { return nil } -func (fs *GcsFs) Open(name string) (*GcsFile, error) { +func (fs *Fs) Open(name string) (*GcsFile, error) { return fs.OpenFile(name, os.O_RDONLY, 0) } -func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) { +func (fs *Fs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) { var file *GcsFile var err error @@ -277,7 +277,7 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile return file, nil } -func (fs *GcsFs) Remove(name string) error { +func (fs *Fs) Remove(name string) error { name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) if err := validateName(name); err != nil { return err @@ -318,7 +318,7 @@ func (fs *GcsFs) Remove(name string) error { return obj.Delete(fs.ctx) } -func (fs *GcsFs) RemoveAll(path string) error { +func (fs *Fs) RemoveAll(path string) error { path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path))) if err := validateName(path); err != nil { return err @@ -351,7 +351,7 @@ func (fs *GcsFs) RemoveAll(path string) error { return fs.Remove(path) } -func (fs *GcsFs) Rename(oldName, newName string) error { +func (fs *Fs) Rename(oldName, newName string) error { oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName))) if err := validateName(oldName); err != nil { return err @@ -378,7 +378,7 @@ func (fs *GcsFs) Rename(oldName, newName string) error { return src.Delete(fs.ctx) } -func (fs *GcsFs) Stat(name string) (os.FileInfo, error) { +func (fs *Fs) Stat(name string) (os.FileInfo, error) { name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) if err := validateName(name); err != nil { return nil, err @@ -387,14 +387,14 @@ func (fs *GcsFs) Stat(name string) (os.FileInfo, error) { return newFileInfo(name, fs, defaultFileMode) } -func (fs *GcsFs) Chmod(_ string, _ os.FileMode) error { +func (fs *Fs) Chmod(_ string, _ os.FileMode) error { return errors.New("method Chmod is not implemented in GCS") } -func (fs *GcsFs) Chtimes(_ string, _, _ time.Time) error { +func (fs *Fs) Chtimes(_ string, _, _ time.Time) error { return errors.New("method Chtimes is not implemented. Create, Delete, Updated times are read only fields in GCS and set implicitly") } -func (fs *GcsFs) Chown(_ string, _, _ int) error { +func (fs *Fs) Chown(_ string, _, _ int) error { return errors.New("method Chown is not implemented for GCS") } diff --git a/gcs-fake-service-account.json b/gcsfs/gcs-fake-service-account.json similarity index 100% rename from gcs-fake-service-account.json rename to gcsfs/gcs-fake-service-account.json diff --git a/gcs.go b/gcsfs/gcs.go similarity index 88% rename from gcs.go rename to gcsfs/gcs.go index 11358221..78cc924f 100644 --- a/gcs.go +++ b/gcsfs/gcs.go @@ -14,29 +14,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -package afero +package gcsfs import ( "context" "os" "time" - "github.com/spf13/afero/gcsfs" - "cloud.google.com/go/storage" "github.com/googleapis/google-cloud-go-testing/storage/stiface" + "github.com/spf13/afero" "google.golang.org/api/option" ) type GcsFs struct { - source *gcsfs.GcsFs + source *Fs } // NewGcsFS creates a GCS file system, automatically instantiating and decorating the storage client. // You can provide additional options to be passed to the client creation, as per // cloud.google.com/go/storage documentation -func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) { +func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (afero.Fs, error) { if json := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON"); json != "" { opts = append(opts, option.WithCredentialsJSON([]byte(json))) } @@ -49,7 +48,7 @@ func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) { } // NewGcsFSWithSeparator is the same as NewGcsFS, but the files system will use the provided folder separator. -func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (Fs, error) { +func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (afero.Fs, error) { client, err := storage.NewClient(ctx, opts...) if err != nil { return nil, err @@ -59,17 +58,17 @@ func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ... } // NewGcsFSFromClient creates a GCS file system from a given storage client -func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (Fs, error) { +func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (afero.Fs, error) { c := stiface.AdaptClient(client) - return &GcsFs{gcsfs.NewGcsFs(ctx, c)}, nil + return &GcsFs{NewGcsFs(ctx, c)}, nil } // NewGcsFSFromClientWithSeparator is the same as NewGcsFSFromClient, but the file system will use the provided folder separator. -func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (Fs, error) { +func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (afero.Fs, error) { c := stiface.AdaptClient(client) - return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil + return &GcsFs{NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil } // Wraps gcs.GcsFs and convert some return types to afero interfaces. @@ -77,7 +76,7 @@ func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client func (fs *GcsFs) Name() string { return fs.source.Name() } -func (fs *GcsFs) Create(name string) (File, error) { +func (fs *GcsFs) Create(name string) (afero.File, error) { return fs.source.Create(name) } func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error { @@ -86,10 +85,10 @@ func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error { func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error { return fs.source.MkdirAll(path, perm) } -func (fs *GcsFs) Open(name string) (File, error) { +func (fs *GcsFs) Open(name string) (afero.File, error) { return fs.source.Open(name) } -func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { +func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { return fs.source.OpenFile(name, flag, perm) } func (fs *GcsFs) Remove(name string) error { diff --git a/gcs_mocks.go b/gcsfs/gcs_mocks.go similarity index 91% rename from gcs_mocks.go rename to gcsfs/gcs_mocks.go index 57e0da56..272eb9f1 100644 --- a/gcs_mocks.go +++ b/gcsfs/gcs_mocks.go @@ -9,7 +9,7 @@ // switching over to a real bucket - and then the mocks have to be adjusted to match the // implementation. -package afero +package gcsfs import ( "context" @@ -17,10 +17,9 @@ import ( "os" "strings" - "github.com/spf13/afero/gcsfs" - "cloud.google.com/go/storage" "github.com/googleapis/google-cloud-go-testing/storage/stiface" + "github.com/spf13/afero" "google.golang.org/api/iterator" ) @@ -31,11 +30,11 @@ func normSeparators(s string) string { type clientMock struct { stiface.Client - fs Fs + fs afero.Fs } func newClientMock() *clientMock { - return &clientMock{fs: NewMemMapFs()} + return &clientMock{fs: afero.NewMemMapFs()} } func (m *clientMock) Bucket(name string) stiface.BucketHandle { @@ -47,7 +46,7 @@ type bucketMock struct { bucketName string - fs Fs + fs afero.Fs } func (m *bucketMock) Attrs(context.Context) (*storage.BucketAttrs, error) { @@ -66,7 +65,7 @@ type objectMock struct { stiface.ObjectHandle name string - fs Fs + fs afero.Fs } func (o *objectMock) NewWriter(_ context.Context) stiface.Writer { @@ -75,7 +74,7 @@ func (o *objectMock) NewWriter(_ context.Context) stiface.Writer { func (o *objectMock) NewRangeReader(_ context.Context, offset, length int64) (stiface.Reader, error) { if o.name == "" { - return nil, gcsfs.ErrEmptyObjectName + return nil, ErrEmptyObjectName } file, err := o.fs.Open(o.name) @@ -104,14 +103,14 @@ func (o *objectMock) NewRangeReader(_ context.Context, offset, length int64) (st func (o *objectMock) Delete(_ context.Context) error { if o.name == "" { - return gcsfs.ErrEmptyObjectName + return ErrEmptyObjectName } return o.fs.Remove(o.name) } func (o *objectMock) Attrs(_ context.Context) (*storage.ObjectAttrs, error) { if o.name == "" { - return nil, gcsfs.ErrEmptyObjectName + return nil, ErrEmptyObjectName } info, err := o.fs.Stat(o.name) @@ -130,7 +129,7 @@ func (o *objectMock) Attrs(_ context.Context) (*storage.ObjectAttrs, error) { if info.IsDir() { // we have to mock it here, because of FileInfo logic - return nil, gcsfs.ErrObjectDoesNotExist + return nil, ErrObjectDoesNotExist } return res, nil @@ -140,14 +139,14 @@ type writerMock struct { stiface.Writer name string - fs Fs + fs afero.Fs - file File + file afero.File } func (w *writerMock) Write(p []byte) (n int, err error) { if w.name == "" { - return 0, gcsfs.ErrEmptyObjectName + return 0, ErrEmptyObjectName } if w.file == nil { @@ -162,7 +161,7 @@ func (w *writerMock) Write(p []byte) (n int, err error) { func (w *writerMock) Close() error { if w.name == "" { - return gcsfs.ErrEmptyObjectName + return ErrEmptyObjectName } if w.file == nil { var err error @@ -187,7 +186,7 @@ func (w *writerMock) Close() error { type readerMock struct { stiface.Reader - file File + file afero.File buf []byte } @@ -212,9 +211,9 @@ type objectItMock struct { stiface.ObjectIterator name string - fs Fs + fs afero.Fs - dir File + dir afero.File infos []*storage.ObjectAttrs } @@ -227,7 +226,7 @@ func (it *objectItMock) Next() (*storage.ObjectAttrs, error) { } var isDir bool - isDir, err = IsDir(it.fs, it.name) + isDir, err = afero.IsDir(it.fs, it.name) if err != nil { return nil, err } diff --git a/gcs_test.go b/gcsfs/gcs_test.go similarity index 98% rename from gcs_test.go rename to gcsfs/gcs_test.go index 37c2cf56..671787c5 100644 --- a/gcs_test.go +++ b/gcsfs/gcs_test.go @@ -3,7 +3,7 @@ // Most of the tests are "derived" from the Afero's own tarfs implementation. // Write-oriented tests and/or checks have been added on top of that -package afero +package gcsfs import ( "context" @@ -19,10 +19,9 @@ import ( "golang.org/x/oauth2/google" - "github.com/spf13/afero/gcsfs" - "cloud.google.com/go/storage" "github.com/googleapis/google-cloud-go-testing/storage/stiface" + "github.com/spf13/afero" ) const ( @@ -62,7 +61,7 @@ var dirs = []struct { {"testDir1", []string{"testFile"}}, } -var gcsAfs *Afero +var gcsAfs *afero.Afero func TestMain(m *testing.M) { ctx := context.Background() @@ -120,7 +119,7 @@ func TestMain(m *testing.M) { mockClient := newClientMock() mockClient.Client = client - gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, mockClient)}} + gcsAfs = &afero.Afero{Fs: &GcsFs{NewGcsFs(ctx, mockClient)}} // Uncomment to use the real, not mocked, client //gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, client)}} @@ -137,7 +136,7 @@ func createFiles(t *testing.T) { if !f.isdir && f.exists { name := filepath.Join(bucketName, f.name) - var freshFile File + var freshFile afero.File freshFile, err = gcsAfs.Create(name) if err != nil { t.Fatalf("failed to create a file \"%s\": %s", f.name, err) @@ -481,7 +480,7 @@ func TestGcsOpenFile(t *testing.T) { file, err := gcsAfs.OpenFile(name, os.O_RDONLY, 0400) if !f.exists { if (f.name != "" && !errors.Is(err, syscall.ENOENT)) || - (f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) { + (f.name == "" && !errors.Is(err, ErrNoBucketInName)) { t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT) } @@ -524,7 +523,7 @@ func TestGcsFsStat(t *testing.T) { fi, err := gcsAfs.Stat(name) if !f.exists { if (f.name != "" && !errors.Is(err, syscall.ENOENT)) || - (f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) { + (f.name == "" && !errors.Is(err, ErrNoBucketInName)) { t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT) } @@ -698,7 +697,7 @@ func TestGcsGlob(t *testing.T) { } for i, prefixedGlob := range prefixedGlobs { - entries, err := Glob(gcsAfs.Fs, prefixedGlob) + entries, err := afero.Glob(gcsAfs.Fs, prefixedGlob) if err != nil { t.Error(err) }