Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

GCS fs: move all gcsfs related implementations to its own package #333

Merged
merged 1 commit into from Dec 28, 2021
Merged

GCS fs: move all gcsfs related implementations to its own package #333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion gcsfs/file.go
View file
Open in desktop
Expand Up @@ -44,7 +44,7 @@ type GcsFile struct {

func NewGcsFile(
ctx context.Context,
fs *GcsFs,
fs *Fs,
obj stiface.ObjectHandle,
openFlags int,
// Unused: there is no use to the file mode in GCloud just yet - but we keep it here, just in case we need it
Expand Down
2 changes: 1 addition & 1 deletion gcsfs/file_info.go
View file
Open in desktop
Expand Up @@ -37,7 +37,7 @@ type FileInfo struct {
fileMode os.FileMode
}

func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error) {
func newFileInfo(name string, fs *Fs, fileMode os.FileMode) (*FileInfo, error) {
res := &FileInfo{
name: name,
size: folderSize,
Expand Down
2 changes: 1 addition & 1 deletion gcsfs/file_resource.go
View file
Open in desktop
Expand Up @@ -40,7 +40,7 @@ const (
type gcsFileResource struct {
ctx context.Context

fs *GcsFs
fs *Fs

obj stiface.ObjectHandle
name string
Expand Down
48 changes: 24 additions & 24 deletions gcsfs/fs.go
View file
Open in desktop
Expand Up @@ -33,8 +33,8 @@ const (
gsPrefix = "gs://"
)

// GcsFs is a Fs implementation that uses functions provided by google cloud storage
type GcsFs struct {
// Fs is a Fs implementation that uses functions provided by google cloud storage
type Fs struct {
ctx context.Context
client stiface.Client
separator string
Expand All @@ -45,12 +45,12 @@ type GcsFs struct {
autoRemoveEmptyFolders bool //trigger for creating "virtual folders" (not required by GCSs)
}

func NewGcsFs(ctx context.Context, client stiface.Client) *GcsFs {
func NewGcsFs(ctx context.Context, client stiface.Client) *Fs {
return NewGcsFsWithSeparator(ctx, client, "/")
}

func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *GcsFs {
return &GcsFs{
func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *Fs {
return &Fs{
ctx: ctx,
client: client,
separator: folderSep,
Expand All @@ -61,17 +61,17 @@ func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep
}

// normSeparators will normalize all "\\" and "/" to the provided separator
func (fs *GcsFs) normSeparators(s string) string {
func (fs *Fs) normSeparators(s string) string {
return strings.Replace(strings.Replace(s, "\\", fs.separator, -1), "/", fs.separator, -1)
}

func (fs *GcsFs) ensureTrailingSeparator(s string) string {
func (fs *Fs) ensureTrailingSeparator(s string) string {
if len(s) > 0 && !strings.HasSuffix(s, fs.separator) {
return s + fs.separator
}
return s
}
func (fs *GcsFs) ensureNoLeadingSeparator(s string) string {
func (fs *Fs) ensureNoLeadingSeparator(s string) string {
if len(s) > 0 && strings.HasPrefix(s, fs.separator) {
s = s[len(fs.separator):]
}
Expand All @@ -94,13 +94,13 @@ func validateName(s string) error {
}

// Splits provided name into bucket name and path
func (fs *GcsFs) splitName(name string) (bucketName string, path string) {
func (fs *Fs) splitName(name string) (bucketName string, path string) {
splitName := strings.Split(name, fs.separator)

return splitName[0], strings.Join(splitName[1:], fs.separator)
}

func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) {
func (fs *Fs) getBucket(name string) (stiface.BucketHandle, error) {
bucket := fs.buckets[name]
if bucket == nil {
bucket = fs.client.Bucket(name)
Expand All @@ -112,7 +112,7 @@ func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) {
return bucket, nil
}

func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) {
func (fs *Fs) getObj(name string) (stiface.ObjectHandle, error) {
bucketName, path := fs.splitName(name)

bucket, err := fs.getBucket(bucketName)
Expand All @@ -123,9 +123,9 @@ func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) {
return bucket.Object(path), nil
}

func (fs *GcsFs) Name() string { return "GcsFs" }
func (fs *Fs) Name() string { return "GcsFs" }

func (fs *GcsFs) Create(name string) (*GcsFile, error) {
func (fs *Fs) Create(name string) (*GcsFile, error) {
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return nil, err
Expand Down Expand Up @@ -156,7 +156,7 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) {
return file, nil
}

func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error {
func (fs *Fs) Mkdir(name string, _ os.FileMode) error {
name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name))))
if err := validateName(name); err != nil {
return err
Expand All @@ -179,7 +179,7 @@ func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error {
return w.Close()
}

func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
func (fs *Fs) MkdirAll(path string, perm os.FileMode) error {
path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path))))
if err := validateName(path); err != nil {
return err
Expand Down Expand Up @@ -216,11 +216,11 @@ func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
return nil
}

func (fs *GcsFs) Open(name string) (*GcsFile, error) {
func (fs *Fs) Open(name string) (*GcsFile, error) {
return fs.OpenFile(name, os.O_RDONLY, 0)
}

func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) {
func (fs *Fs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) {
var file *GcsFile
var err error

Expand Down Expand Up @@ -277,7 +277,7 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile
return file, nil
}

func (fs *GcsFs) Remove(name string) error {
func (fs *Fs) Remove(name string) error {
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return err
Expand Down Expand Up @@ -318,7 +318,7 @@ func (fs *GcsFs) Remove(name string) error {
return obj.Delete(fs.ctx)
}

func (fs *GcsFs) RemoveAll(path string) error {
func (fs *Fs) RemoveAll(path string) error {
path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path)))
if err := validateName(path); err != nil {
return err
Expand Down Expand Up @@ -351,7 +351,7 @@ func (fs *GcsFs) RemoveAll(path string) error {
return fs.Remove(path)
}

func (fs *GcsFs) Rename(oldName, newName string) error {
func (fs *Fs) Rename(oldName, newName string) error {
oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName)))
if err := validateName(oldName); err != nil {
return err
Expand All @@ -378,7 +378,7 @@ func (fs *GcsFs) Rename(oldName, newName string) error {
return src.Delete(fs.ctx)
}

func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
func (fs *Fs) Stat(name string) (os.FileInfo, error) {
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return nil, err
Expand All @@ -387,14 +387,14 @@ func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
return newFileInfo(name, fs, defaultFileMode)
}

func (fs *GcsFs) Chmod(_ string, _ os.FileMode) error {
func (fs *Fs) Chmod(_ string, _ os.FileMode) error {
return errors.New("method Chmod is not implemented in GCS")
}

func (fs *GcsFs) Chtimes(_ string, _, _ time.Time) error {
func (fs *Fs) Chtimes(_ string, _, _ time.Time) error {
return errors.New("method Chtimes is not implemented. Create, Delete, Updated times are read only fields in GCS and set implicitly")
}

func (fs *GcsFs) Chown(_ string, _, _ int) error {
func (fs *Fs) Chown(_ string, _, _ int) error {
return errors.New("method Chown is not implemented for GCS")
}
0 gcs-fake-service-account.json → gcsfs/gcs-fake-service-account.json
View file
Open in desktop
File renamed without changes.
25 changes: 12 additions & 13 deletions gcs.go → gcsfs/gcs.go
View file
Open in desktop
Expand Up @@ -14,29 +14,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package afero
package gcsfs

import (
"context"
"os"
"time"

"github.com/spf13/afero/gcsfs"

"cloud.google.com/go/storage"
"github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/spf13/afero"

"google.golang.org/api/option"
)

type GcsFs struct {
source *gcsfs.GcsFs
source *Fs
}

// NewGcsFS creates a GCS file system, automatically instantiating and decorating the storage client.
// You can provide additional options to be passed to the client creation, as per
// cloud.google.com/go/storage documentation
func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (afero.Fs, error) {
if json := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON"); json != "" {
opts = append(opts, option.WithCredentialsJSON([]byte(json)))
}
Expand All @@ -49,7 +48,7 @@ func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
}

// NewGcsFSWithSeparator is the same as NewGcsFS, but the files system will use the provided folder separator.
func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (Fs, error) {
func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (afero.Fs, error) {
client, err := storage.NewClient(ctx, opts...)
if err != nil {
return nil, err
Expand All @@ -59,25 +58,25 @@ func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...
}

// NewGcsFSFromClient creates a GCS file system from a given storage client
func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (Fs, error) {
func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (afero.Fs, error) {
c := stiface.AdaptClient(client)

return &GcsFs{gcsfs.NewGcsFs(ctx, c)}, nil
return &GcsFs{NewGcsFs(ctx, c)}, nil
}

// NewGcsFSFromClientWithSeparator is the same as NewGcsFSFromClient, but the file system will use the provided folder separator.
func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (Fs, error) {
func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (afero.Fs, error) {
c := stiface.AdaptClient(client)

return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
return &GcsFs{NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
}

// Wraps gcs.GcsFs and convert some return types to afero interfaces.

func (fs *GcsFs) Name() string {
return fs.source.Name()
}
func (fs *GcsFs) Create(name string) (File, error) {
func (fs *GcsFs) Create(name string) (afero.File, error) {
return fs.source.Create(name)
}
func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
Expand All @@ -86,10 +85,10 @@ func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
return fs.source.MkdirAll(path, perm)
}
func (fs *GcsFs) Open(name string) (File, error) {
func (fs *GcsFs) Open(name string) (afero.File, error) {
return fs.source.Open(name)
}
func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
return fs.source.OpenFile(name, flag, perm)
}
func (fs *GcsFs) Remove(name string) error {
Expand Down