diff --git a/gcs.go b/gcs.go index a9787350..fd1f7175 100644 --- a/gcs.go +++ b/gcs.go @@ -21,9 +21,11 @@ import ( "os" "time" + "github.com/spf13/afero/gcsfs" + "cloud.google.com/go/storage" "github.com/googleapis/google-cloud-go-testing/storage/stiface" - "github.com/spf13/afero/gcsfs" + "google.golang.org/api/option" ) @@ -31,47 +33,44 @@ type GcsFs struct { source *gcsfs.GcsFs } -// Creates a GCS file system, automatically instantiating and decorating the storage client. +// NewGcsFS creates a GCS file system, automatically instantiating and decorating the storage client. // You can provide additional options to be passed to the client creation, as per // cloud.google.com/go/storage documentation -func NewGcsFS(ctx context.Context, bucketName string, opts ...option.ClientOption) (Fs, error) { +func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) { client, err := storage.NewClient(ctx, opts...) if err != nil { return nil, err } - return NewGcsFSFromClient(ctx, client, bucketName) + return NewGcsFSFromClient(ctx, client) } -// The same as NewGcsFS, but the files system will use the provided folder separator. -func NewGcsFSWithSeparator(ctx context.Context, bucketName, folderSeparator string, opts ...option.ClientOption) (Fs, error) { +// NewGcsFSWithSeparator is the same as NewGcsFS, but the files system will use the provided folder separator. +func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (Fs, error) { client, err := storage.NewClient(ctx, opts...) if err != nil { return nil, err } - return NewGcsFSFromClientWithSeparator(ctx, client, bucketName, folderSeparator) + return NewGcsFSFromClientWithSeparator(ctx, client, folderSeparator) } -// Creates a GCS file system from a given storage client -func NewGcsFSFromClient(ctx context.Context, client *storage.Client, bucketName string) (Fs, error) { +// NewGcsFSFromClient creates a GCS file system from a given storage client +func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (Fs, error) { c := stiface.AdaptClient(client) - bucket := c.Bucket(bucketName) - - return &GcsFs{gcsfs.NewGcsFs(ctx, bucket)}, nil + return &GcsFs{gcsfs.NewGcsFs(ctx, c)}, nil } -// Same as NewGcsFSFromClient, but the file system will use the provided folder separator. -func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, bucketName, folderSeparator string) (Fs, error) { +// NewGcsFSFromClientWithSeparator is the same as NewGcsFSFromClient, but the file system will use the provided folder separator. +func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (Fs, error) { c := stiface.AdaptClient(client) - bucket := c.Bucket(bucketName) - - return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, bucket, folderSeparator)}, nil + return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil } // Wraps gcs.GcsFs and convert some return types to afero interfaces. + func (fs *GcsFs) Name() string { return fs.source.Name() } diff --git a/gcs_mocks.go b/gcs_mocks.go index 2d6928e3..acbb1e5b 100644 --- a/gcs_mocks.go +++ b/gcs_mocks.go @@ -50,6 +50,10 @@ type bucketMock struct { fs Fs } +func (m *bucketMock) Attrs(context.Context) (*storage.BucketAttrs, error) { + return &storage.BucketAttrs{}, nil +} + func (m *bucketMock) Object(name string) stiface.ObjectHandle { return &objectMock{name: name, fs: m.fs} } @@ -193,6 +197,10 @@ func (r *readerMock) Read(p []byte) (int, error) { return r.file.Read(p) } +func (r *readerMock) Close() error { + return r.file.Close() +} + type objectItMock struct { stiface.ObjectIterator diff --git a/gcs_test.go b/gcs_test.go index 3b701e1f..4e3d75f5 100644 --- a/gcs_test.go +++ b/gcs_test.go @@ -17,6 +17,8 @@ import ( "syscall" "testing" + "golang.org/x/oauth2/google" + "github.com/spf13/afero/gcsfs" "cloud.google.com/go/storage" @@ -28,6 +30,8 @@ const ( dirSize = 42 ) +var bucketName = "a-test-bucket" + var files = []struct { name string exists bool @@ -37,13 +41,14 @@ var files = []struct { offset int64 contentAtOffset string }{ - {"", true, true, dirSize, "", 0, ""}, // this is NOT a valid path for GCS, so we do some magic here {"sub", true, true, dirSize, "", 0, ""}, {"sub/testDir2", true, true, dirSize, "", 0, ""}, {"sub/testDir2/testFile", true, false, 8 * 1024, "c", 4 * 1024, "d"}, {"testFile", true, false, 12 * 1024, "a", 7 * 1024, "b"}, {"testDir1/testFile", true, false, 3 * 512, "b", 512, "c"}, + {"", false, true, dirSize, "", 0, ""}, // special case + {"nonExisting", false, false, dirSize, "", 0, ""}, } @@ -51,7 +56,7 @@ var dirs = []struct { name string children []string }{ - {"", []string{"sub", "testDir1", "testFile"}}, + {"", []string{"sub", "testDir1", "testFile"}}, // in this case it will be prepended with bucket name {"sub", []string{"testDir2"}}, {"sub/testDir2", []string{"testFile"}}, {"testDir1", []string{"testFile"}}, @@ -63,20 +68,30 @@ func TestMain(m *testing.M) { ctx := context.Background() var err error - // Check if GOOGLE_APPLICATION_CREDENTIALS are present. If not, then a fake service account + // in order to respect deferring + var exitCode int + defer os.Exit(exitCode) + + // Check if any credentials are present. If not, a fake service account, taken from the link // would be used: https://github.com/google/oauth2l/blob/master/integration/fixtures/fake-service-account.json - if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { + cred, err := google.FindDefaultCredentials(ctx) + if err != nil && !strings.HasPrefix(err.Error(), "google: could not find default credentials") { + exitCode = 2 + panic(err) + } + + if cred == nil { var fakeCredentialsAbsPath string fakeCredentialsAbsPath, err = filepath.Abs("gcs-fake-service-account.json") if err != nil { - fmt.Print(err) - os.Exit(1) + exitCode = 2 + panic(err) } err = os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", fakeCredentialsAbsPath) if err != nil { - fmt.Print(err) - os.Exit(1) + exitCode = 2 + panic(err) } // reset it after the run @@ -92,8 +107,8 @@ func TestMain(m *testing.M) { var c *storage.Client c, err = storage.NewClient(ctx) if err != nil { - fmt.Print(err) - os.Exit(1) + exitCode = 2 + panic(err) } client := stiface.AdaptClient(c) @@ -101,18 +116,12 @@ func TestMain(m *testing.M) { mockClient := newClientMock() mockClient.Client = client - bucket := mockClient.Bucket("a-test-bucket") + gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, mockClient)}} - // If you want to run the test suite on a LIVE bucket, comment the previous - // block and uncomment the line below and put your bucket name there. - // Keep in mind, that GCS will likely rate limit you, so it would be impossible - // to run the entire suite at once, only test by test. - //bucket := client.Bucket("a-test-bucket") + // Uncomment to use the real, not mocked, client + //gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, client)}} - gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, bucket)}} - - // defer here to assure our Env cleanup happens, if the mock was used - defer os.Exit(m.Run()) + exitCode = m.Run() } func createFiles(t *testing.T) { @@ -122,8 +131,10 @@ func createFiles(t *testing.T) { // the files have to be created first for _, f := range files { if !f.isdir && f.exists { + name := filepath.Join(bucketName, f.name) + var freshFile File - freshFile, err = gcsAfs.Create(f.name) + freshFile, err = gcsAfs.Create(name) if err != nil { t.Fatalf("failed to create a file \"%s\": %s", f.name, err) } @@ -160,7 +171,9 @@ func removeFiles(t *testing.T) { // the files have to be created first for _, f := range files { if !f.isdir && f.exists { - err = gcsAfs.Remove(f.name) + name := filepath.Join(bucketName, f.name) + + err = gcsAfs.Remove(name) if err != nil && err == syscall.ENOENT { t.Errorf("failed to remove file \"%s\": %s", f.name, err) } @@ -168,43 +181,55 @@ func removeFiles(t *testing.T) { } } -func TestFsOpen(t *testing.T) { +func TestGcsFsOpen(t *testing.T) { createFiles(t) defer removeFiles(t) for _, f := range files { - file, err := gcsAfs.Open(f.name) - if (err == nil) != f.exists { - t.Errorf("%v exists = %v, but got err = %v", f.name, f.exists, err) - } + nameBase := filepath.Join(bucketName, f.name) - if !f.exists { - continue + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - if err != nil { - t.Fatalf("%v: %v", f.name, err) + if f.name == "" { + names = []string{f.name} } - if file.Name() != filepath.FromSlash(f.name) { - t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(f.name)) - } + for _, name := range names { + file, err := gcsAfs.Open(name) + if (err == nil) != f.exists { + t.Errorf("%v exists = %v, but got err = %v", name, f.exists, err) + } - s, err := file.Stat() - if err != nil { - t.Fatalf("stat %v: got error '%v'", file.Name(), err) - } + if !f.exists { + continue + } + if err != nil { + t.Fatalf("%v: %v", name, err) + } - if isdir := s.IsDir(); isdir != f.isdir { - t.Errorf("%v directory, got: %v, expected: %v", file.Name(), isdir, f.isdir) - } + if file.Name() != filepath.FromSlash(nameBase) { + t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(nameBase)) + } + + s, err := file.Stat() + if err != nil { + t.Fatalf("stat %v: got error '%v'", file.Name(), err) + } + + if isdir := s.IsDir(); isdir != f.isdir { + t.Errorf("%v directory, got: %v, expected: %v", file.Name(), isdir, f.isdir) + } - if size := s.Size(); size != f.size { - t.Errorf("%v size, got: %v, expected: %v", file.Name(), size, f.size) + if size := s.Size(); size != f.size { + t.Errorf("%v size, got: %v, expected: %v", file.Name(), size, f.size) + } } } } -func TestRead(t *testing.T) { +func TestGcsRead(t *testing.T) { createFiles(t) defer removeFiles(t) @@ -213,25 +238,36 @@ func TestRead(t *testing.T) { continue } - file, err := gcsAfs.Open(f.name) - if err != nil { - t.Fatalf("opening %v: %v", f.name, err) + nameBase := filepath.Join(bucketName, f.name) + + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, + } + if f.name == "" { + names = []string{f.name} } - buf := make([]byte, 8) - n, err := file.Read(buf) - if err != nil { - if f.isdir && (err != syscall.EISDIR) { - t.Errorf("%v got error %v, expected EISDIR", f.name, err) - } else if !f.isdir { - t.Errorf("%v: %v", f.name, err) + for _, name := range names { + file, err := gcsAfs.Open(name) + if err != nil { + t.Fatalf("opening %v: %v", name, err) } - } else if n != 8 { - t.Errorf("%v: got %d read bytes, expected 8", f.name, n) - } else if string(buf) != strings.Repeat(f.content, testBytes) { - t.Errorf("%v: got <%s>, expected <%s>", f.name, f.content, string(buf)) - } + buf := make([]byte, 8) + n, err := file.Read(buf) + if err != nil { + if f.isdir && (err != syscall.EISDIR) { + t.Errorf("%v got error %v, expected EISDIR", name, err) + } else if !f.isdir { + t.Errorf("%v: %v", name, err) + } + } else if n != 8 { + t.Errorf("%v: got %d read bytes, expected 8", name, n) + } else if string(buf) != strings.Repeat(f.content, testBytes) { + t.Errorf("%v: got <%s>, expected <%s>", f.name, f.content, string(buf)) + } + } } } @@ -244,25 +280,36 @@ func TestGcsReadAt(t *testing.T) { continue } - file, err := gcsAfs.Open(f.name) - if err != nil { - t.Fatalf("opening %v: %v", f.name, err) + nameBase := filepath.Join(bucketName, f.name) + + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, + } + if f.name == "" { + names = []string{f.name} } - buf := make([]byte, testBytes) - n, err := file.ReadAt(buf, f.offset-testBytes/2) - if err != nil { - if f.isdir && (err != syscall.EISDIR) { - t.Errorf("%v got error %v, expected EISDIR", f.name, err) - } else if !f.isdir { - t.Errorf("%v: %v", f.name, err) + for _, name := range names { + file, err := gcsAfs.Open(name) + if err != nil { + t.Fatalf("opening %v: %v", name, err) } - } else if n != 8 { - t.Errorf("%v: got %d read bytes, expected 8", f.name, n) - } else if string(buf) != strings.Repeat(f.content, testBytes/2)+strings.Repeat(f.contentAtOffset, testBytes/2) { - t.Errorf("%v: got <%s>, expected <%s>", f.name, f.contentAtOffset, string(buf)) - } + buf := make([]byte, testBytes) + n, err := file.ReadAt(buf, f.offset-testBytes/2) + if err != nil { + if f.isdir && (err != syscall.EISDIR) { + t.Errorf("%v got error %v, expected EISDIR", name, err) + } else if !f.isdir { + t.Errorf("%v: %v", name, err) + } + } else if n != 8 { + t.Errorf("%v: got %d read bytes, expected 8", f.name, n) + } else if string(buf) != strings.Repeat(f.content, testBytes/2)+strings.Repeat(f.contentAtOffset, testBytes/2) { + t.Errorf("%v: got <%s>, expected <%s>", f.name, f.contentAtOffset, string(buf)) + } + } } } @@ -275,43 +322,55 @@ func TestGcsSeek(t *testing.T) { continue } - file, err := gcsAfs.Open(f.name) - if err != nil { - t.Fatalf("opening %v: %v", f.name, err) - } + nameBase := filepath.Join(bucketName, f.name) - var tests = []struct { - offIn int64 - whence int - offOut int64 - }{ - {0, io.SeekStart, 0}, - {10, io.SeekStart, 10}, - {1, io.SeekCurrent, 11}, - {10, io.SeekCurrent, 21}, - {0, io.SeekEnd, f.size}, - {-1, io.SeekEnd, f.size - 1}, + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, + } + if f.name == "" { + names = []string{f.name} } - for _, s := range tests { - n, err := file.Seek(s.offIn, s.whence) + for _, name := range names { + file, err := gcsAfs.Open(name) if err != nil { - if f.isdir && err == syscall.EISDIR { - continue - } + t.Fatalf("opening %v: %v", name, err) + } - t.Errorf("%v: %v", f.name, err) + var tests = []struct { + offIn int64 + whence int + offOut int64 + }{ + {0, io.SeekStart, 0}, + {10, io.SeekStart, 10}, + {1, io.SeekCurrent, 11}, + {10, io.SeekCurrent, 21}, + {0, io.SeekEnd, f.size}, + {-1, io.SeekEnd, f.size - 1}, } - if n != s.offOut { - t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offIn, s.whence, n, s.offOut) + for _, s := range tests { + n, err := file.Seek(s.offIn, s.whence) + if err != nil { + if f.isdir && err == syscall.EISDIR { + continue + } + + t.Errorf("%v: %v", name, err) + } + + if n != s.offOut { + t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offIn, s.whence, n, s.offOut) + } } } } } -func TestName(t *testing.T) { +func TestGcsName(t *testing.T) { createFiles(t) defer removeFiles(t) @@ -320,20 +379,32 @@ func TestName(t *testing.T) { continue } - file, err := gcsAfs.Open(f.name) - if err != nil { - t.Fatalf("opening %v: %v", f.name, err) + nameBase := filepath.Join(bucketName, f.name) + + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, + } + if f.name == "" { + names = []string{f.name} } - n := file.Name() - if n != filepath.FromSlash(f.name) { - t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(f.name)) + for _, name := range names { + file, err := gcsAfs.Open(name) + if err != nil { + t.Fatalf("opening %v: %v", name, err) + } + + n := file.Name() + if n != filepath.FromSlash(nameBase) { + t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(nameBase)) + } } } } -func TestClose(t *testing.T) { +func TestGcsClose(t *testing.T) { createFiles(t) defer removeFiles(t) @@ -342,35 +413,47 @@ func TestClose(t *testing.T) { continue } - file, err := gcsAfs.Open(f.name) - if err != nil { - t.Fatalf("opening %v: %v", f.name, err) - } + nameBase := filepath.Join(bucketName, f.name) - err = file.Close() - if err != nil { - t.Errorf("%v: %v", f.name, err) + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - - err = file.Close() - if err == nil { - t.Errorf("%v: closing twice should return an error", f.name) + if f.name == "" { + names = []string{f.name} } - buf := make([]byte, 8) - n, err := file.Read(buf) - if n != 0 || err == nil { - t.Errorf("%v: could read from a closed file", f.name) - } + for _, name := range names { + file, err := gcsAfs.Open(name) + if err != nil { + t.Fatalf("opening %v: %v", name, err) + } - n, err = file.ReadAt(buf, 256) - if n != 0 || err == nil { - t.Errorf("%v: could readAt from a closed file", f.name) - } + err = file.Close() + if err != nil { + t.Errorf("%v: %v", name, err) + } + + err = file.Close() + if err == nil { + t.Errorf("%v: closing twice should return an error", name) + } + + buf := make([]byte, 8) + n, err := file.Read(buf) + if n != 0 || err == nil { + t.Errorf("%v: could read from a closed file", name) + } + + n, err = file.ReadAt(buf, 256) + if n != 0 || err == nil { + t.Errorf("%v: could readAt from a closed file", name) + } - off, err := file.Seek(0, io.SeekStart) - if off != 0 || err == nil { - t.Errorf("%v: could seek from a closed file", f.name) + off, err := file.Seek(0, io.SeekStart) + if off != 0 || err == nil { + t.Errorf("%v: could seek from a closed file", name) + } } } } @@ -380,56 +463,81 @@ func TestGcsOpenFile(t *testing.T) { defer removeFiles(t) for _, f := range files { - file, err := gcsAfs.OpenFile(f.name, os.O_RDONLY, 0400) - if !f.exists { - if !errors.Is(err, syscall.ENOENT) { - t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT) - } + nameBase := filepath.Join(bucketName, f.name) - continue + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - - if err != nil { - t.Fatalf("%v: %v", f.name, err) + if f.name == "" { + names = []string{f.name} } - err = file.Close() - if err != nil { - t.Fatalf("failed to close a file \"%s\": %s", f.name, err) - } + for _, name := range names { + file, err := gcsAfs.OpenFile(name, os.O_RDONLY, 0400) + if !f.exists { + if (f.name != "" && !errors.Is(err, syscall.ENOENT)) || + (f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) { + t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT) + } - file, err = gcsAfs.OpenFile(f.name, os.O_CREATE, 0600) - if !errors.Is(err, syscall.EPERM) { - t.Errorf("%v: open for write: got %v, expected %v", f.name, err, syscall.EPERM) - } + continue + } + if err != nil { + t.Fatalf("%v: %v", name, err) + } + + err = file.Close() + if err != nil { + t.Fatalf("failed to close a file \"%s\": %s", name, err) + } + + file, err = gcsAfs.OpenFile(name, os.O_CREATE, 0600) + if !errors.Is(err, syscall.EPERM) { + t.Errorf("%v: open for write: got %v, expected %v", name, err, syscall.EPERM) + } + } } } -func TestFsStat(t *testing.T) { +func TestGcsFsStat(t *testing.T) { createFiles(t) defer removeFiles(t) for _, f := range files { - fi, err := gcsAfs.Stat(f.name) - if !f.exists { - if !errors.Is(err, syscall.ENOENT) { - t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT) - } + nameBase := filepath.Join(bucketName, f.name) - continue + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - - if err != nil { - t.Fatalf("stat %v: got error '%v'", f.name, err) + if f.name == "" { + names = []string{f.name} } - if isdir := fi.IsDir(); isdir != f.isdir { - t.Errorf("%v directory, got: %v, expected: %v", f.name, isdir, f.isdir) - } + for _, name := range names { + fi, err := gcsAfs.Stat(name) + if !f.exists { + if (f.name != "" && !errors.Is(err, syscall.ENOENT)) || + (f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) { + t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT) + } - if size := fi.Size(); size != f.size { - t.Errorf("%v size, got: %v, expected: %v", f.name, size, f.size) + continue + } + + if err != nil { + t.Fatalf("stat %v: got error '%v'", name, err) + } + + if isdir := fi.IsDir(); isdir != f.isdir { + t.Errorf("%v directory, got: %v, expected: %v", name, isdir, f.isdir) + } + + if size := fi.Size(); size != f.size { + t.Errorf("%v size, got: %v, expected: %v", name, size, f.size) + } } } } @@ -439,47 +547,65 @@ func TestGcsReaddir(t *testing.T) { defer removeFiles(t) for _, d := range dirs { - dir, err := gcsAfs.Open(d.name) - if err != nil { - t.Fatal(err) - } + nameBase := filepath.Join(bucketName, d.name) - fi, err := dir.Readdir(0) - if err != nil { - t.Fatal(err) - } - var names []string - for _, f := range fi { - names = append(names, f.Name()) + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - if !reflect.DeepEqual(names, d.children) { - t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children) - } + for _, name := range names { + dir, err := gcsAfs.Open(name) + if err != nil { + t.Fatal(err) + } - fi, err = dir.Readdir(1) - if err != nil { - t.Fatal(err) - } + fi, err := dir.Readdir(0) + if err != nil { + t.Fatal(err) + } + var fileNames []string + for _, f := range fi { + fileNames = append(fileNames, f.Name()) + } - names = []string{} - for _, f := range fi { - names = append(names, f.Name()) - } + if !reflect.DeepEqual(fileNames, d.children) { + t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children) + } - if !reflect.DeepEqual(names, d.children[0:1]) { - t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1]) + fi, err = dir.Readdir(1) + if err != nil { + t.Fatal(err) + } + + fileNames = []string{} + for _, f := range fi { + fileNames = append(fileNames, f.Name()) + } + + if !reflect.DeepEqual(fileNames, d.children[0:1]) { + t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children[0:1]) + } } } - dir, err := gcsAfs.Open("testFile") - if err != nil { - t.Fatal(err) + nameBase := filepath.Join(bucketName, "testFile") + + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - _, err = dir.Readdir(-1) - if err != syscall.ENOTDIR { - t.Fatal("Expected error") + for _, name := range names { + dir, err := gcsAfs.Open(name) + if err != nil { + t.Fatal(err) + } + + _, err = dir.Readdir(-1) + if err != syscall.ENOTDIR { + t.Fatal("Expected error") + } } } @@ -488,38 +614,56 @@ func TestGcsReaddirnames(t *testing.T) { defer removeFiles(t) for _, d := range dirs { - dir, err := gcsAfs.Open(d.name) - if err != nil { - t.Fatal(err) - } + nameBase := filepath.Join(bucketName, d.name) - names, err := dir.Readdirnames(0) - if err != nil { - t.Fatal(err) + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - if !reflect.DeepEqual(names, d.children) { - t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children) - } + for _, name := range names { + dir, err := gcsAfs.Open(name) + if err != nil { + t.Fatal(err) + } - names, err = dir.Readdirnames(1) - if err != nil { - t.Fatal(err) - } + fileNames, err := dir.Readdirnames(0) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(fileNames, d.children) { + t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children) + } - if !reflect.DeepEqual(names, d.children[0:1]) { - t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1]) + fileNames, err = dir.Readdirnames(1) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(fileNames, d.children[0:1]) { + t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children[0:1]) + } } } - dir, err := gcsAfs.Open("testFile") - if err != nil { - t.Fatal(err) + nameBase := filepath.Join(bucketName, "testFile") + + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - _, err = dir.Readdir(-1) - if err != syscall.ENOTDIR { - t.Fatal("Expected error") + for _, name := range names { + dir, err := gcsAfs.Open(name) + if err != nil { + t.Fatal(err) + } + + _, err = dir.Readdirnames(-1) + if err != syscall.ENOTDIR { + t.Fatal("Expected error") + } } } @@ -531,29 +675,40 @@ func TestGcsGlob(t *testing.T) { glob string entries []string }{ - {filepath.FromSlash("/*"), []string{filepath.FromSlash("/sub"), filepath.FromSlash("/testDir1"), filepath.FromSlash("/testFile")}}, {filepath.FromSlash("*"), []string{filepath.FromSlash("sub"), filepath.FromSlash("testDir1"), filepath.FromSlash("testFile")}}, - {filepath.FromSlash("/sub/*"), []string{filepath.FromSlash("/sub/testDir2")}}, - {filepath.FromSlash("/sub/testDir2/*"), []string{filepath.FromSlash("/sub/testDir2/testFile")}}, - {filepath.FromSlash("/testDir1/*"), []string{filepath.FromSlash("/testDir1/testFile")}}, {filepath.FromSlash("sub/*"), []string{filepath.FromSlash("sub/testDir2")}}, {filepath.FromSlash("sub/testDir2/*"), []string{filepath.FromSlash("sub/testDir2/testFile")}}, {filepath.FromSlash("testDir1/*"), []string{filepath.FromSlash("testDir1/testFile")}}, } { - entries, err := Glob(gcsAfs.Fs, s.glob) - if err != nil { - t.Error(err) + nameBase := filepath.Join(bucketName, s.glob) + + prefixedGlobs := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - if reflect.DeepEqual(entries, s.entries) { - t.Logf("glob: %s: glob ok", s.glob) - } else { - t.Errorf("glob: %s: got %#v, expected %#v", s.glob, entries, s.entries) + + prefixedEntries := [][]string{{}, {}} + for _, entry := range s.entries { + prefixedEntries[0] = append(prefixedEntries[0], filepath.Join(bucketName, entry)) + prefixedEntries[1] = append(prefixedEntries[1], string(os.PathSeparator)+filepath.Join(bucketName, entry)) + } + + for i, prefixedGlob := range prefixedGlobs { + entries, err := Glob(gcsAfs.Fs, prefixedGlob) + if err != nil { + t.Error(err) + } + if reflect.DeepEqual(entries, prefixedEntries[i]) { + t.Logf("glob: %s: glob ok", prefixedGlob) + } else { + t.Errorf("glob: %s: got %#v, expected %#v", prefixedGlob, entries, prefixedEntries) + } } } } -func TestMkdir(t *testing.T) { - dirName := "/a-test-dir" +func TestGcsMkdir(t *testing.T) { + dirName := filepath.Join(bucketName, "a-test-dir") var err error err = gcsAfs.Mkdir(dirName, 0755) @@ -582,45 +737,47 @@ func TestMkdir(t *testing.T) { } } -func TestMkdirAll(t *testing.T) { - err := gcsAfs.MkdirAll("/a/b/c", 0755) +func TestGcsMkdirAll(t *testing.T) { + dirName := filepath.Join(bucketName, "a/b/c") + + err := gcsAfs.MkdirAll(dirName, 0755) if err != nil { t.Fatal(err) } - info, err := gcsAfs.Stat("/a") + info, err := gcsAfs.Stat(filepath.Join(bucketName, "a")) if err != nil { t.Fatal(err) } if !info.Mode().IsDir() { - t.Error("/a: mode is not directory") + t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a")) } if info.Mode() != os.ModeDir|0755 { - t.Errorf("/a: wrong permissions, expected drwxr-xr-x, got %s", info.Mode()) + t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a"), info.Mode()) } - info, err = gcsAfs.Stat("/a/b") + info, err = gcsAfs.Stat(filepath.Join(bucketName, "a/b")) if err != nil { t.Fatal(err) } if !info.Mode().IsDir() { - t.Error("/a/b: mode is not directory") + t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a/b")) } if info.Mode() != os.ModeDir|0755 { - t.Errorf("/a/b: wrong permissions, expected drwxr-xr-x, got %s", info.Mode()) + t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a/b"), info.Mode()) } - info, err = gcsAfs.Stat("/a/b/c") + info, err = gcsAfs.Stat(dirName) if err != nil { t.Fatal(err) } if !info.Mode().IsDir() { - t.Error("/a/b/c: mode is not directory") + t.Errorf("%s: mode is not directory", dirName) } if info.Mode() != os.ModeDir|0755 { - t.Errorf("/a/b/c: wrong permissions, expected drwxr-xr-x, got %s", info.Mode()) + t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode()) } - err = gcsAfs.RemoveAll("/a") + err = gcsAfs.RemoveAll(filepath.Join(bucketName, "a")) if err != nil { - t.Fatalf("failed to remove the folder /a with error: %s", err) + t.Fatalf("failed to remove the folder %s with error: %s", filepath.Join(bucketName, "a"), err) } } diff --git a/gcsfs/errors.go b/gcsfs/errors.go index 7468ad5b..201cd676 100644 --- a/gcsfs/errors.go +++ b/gcsfs/errors.go @@ -22,6 +22,7 @@ import ( ) var ( + ErrNoBucketInName = errors.New("no bucket name found in the name") ErrFileClosed = errors.New("file is closed") ErrOutOfRange = errors.New("out of range") ErrObjectDoesNotExist = errors.New("storage: object doesn't exist") diff --git a/gcsfs/file.go b/gcsfs/file.go index 33364a4a..b3b3d892 100644 --- a/gcsfs/file.go +++ b/gcsfs/file.go @@ -195,11 +195,13 @@ func (o *GcsFile) readdirImpl(count int) ([]*FileInfo, error) { return nil, syscall.ENOTDIR } - path := o.resource.fs.ensureTrailingSeparator(o.Name()) + path := o.resource.fs.ensureTrailingSeparator(o.resource.name) if o.ReadDirIt == nil { //log.Printf("Querying path : %s\n", path) - o.ReadDirIt = o.resource.fs.bucket.Objects( - o.resource.ctx, &storage.Query{Delimiter: o.resource.fs.separator, Prefix: path, Versions: false}) + bucketName, bucketPath := o.resource.fs.splitName(path) + + o.ReadDirIt = o.resource.fs.client.Bucket(bucketName).Objects( + o.resource.ctx, &storage.Query{Delimiter: o.resource.fs.separator, Prefix: bucketPath, Versions: false}) } var res []*FileInfo for { @@ -283,7 +285,7 @@ func (o *GcsFile) Stat() (os.FileInfo, error) { return nil, err } - return newFileInfo(o.Name(), o.resource.fs, o.resource.fileMode) + return newFileInfo(o.resource.name, o.resource.fs, o.resource.fileMode) } func (o *GcsFile) Sync() error { diff --git a/gcsfs/file_info.go b/gcsfs/file_info.go index b14a5f6a..2cccb7ca 100644 --- a/gcsfs/file_info.go +++ b/gcsfs/file_info.go @@ -46,7 +46,10 @@ func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error fileMode: fileMode, } - obj := fs.getObj(name) + obj, err := fs.getObj(name) + if err != nil { + return nil, err + } objAttrs, err := obj.Attrs(fs.ctx) if err != nil { @@ -58,8 +61,9 @@ func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error } else if err.Error() == ErrObjectDoesNotExist.Error() { // Folders do not actually "exist" in GCloud, so we have to check, if something exists with // such a prefix - it := fs.bucket.Objects( - fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: name, Versions: false}) + bucketName, bucketPath := fs.splitName(name) + it := fs.client.Bucket(bucketName).Objects( + fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: bucketPath, Versions: false}) if _, err = it.Next(); err == nil { res.name = fs.ensureTrailingSeparator(res.name) res.isDir = true @@ -100,7 +104,7 @@ func newFileInfoFromAttrs(objAttrs *storage.ObjectAttrs, fileMode os.FileMode) * } func (fi *FileInfo) Name() string { - return filepath.Base(fi.name) + return filepath.Base(filepath.FromSlash(fi.name)) } func (fi *FileInfo) Size() int64 { diff --git a/gcsfs/fs.go b/gcsfs/fs.go index 79bfa3b7..ca452184 100644 --- a/gcsfs/fs.go +++ b/gcsfs/fs.go @@ -19,7 +19,6 @@ package gcsfs import ( "context" "errors" - "log" "os" "path/filepath" "strings" @@ -31,26 +30,29 @@ import ( const ( defaultFileMode = 0755 + gsPrefix = "gs://" ) // GcsFs is a Fs implementation that uses functions provided by google cloud storage type GcsFs struct { - ctx context.Context - bucket stiface.BucketHandle - separator string + ctx context.Context + client stiface.Client + separator string + + buckets map[string]stiface.BucketHandle rawGcsObjects map[string]*GcsFile autoRemoveEmptyFolders bool //trigger for creating "virtual folders" (not required by GCSs) } -func NewGcsFs(ctx context.Context, bucket stiface.BucketHandle) *GcsFs { - return NewGcsFsWithSeparator(ctx, bucket, "/") +func NewGcsFs(ctx context.Context, client stiface.Client) *GcsFs { + return NewGcsFsWithSeparator(ctx, client, "/") } -func NewGcsFsWithSeparator(ctx context.Context, bucket stiface.BucketHandle, folderSep string) *GcsFs { +func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *GcsFs { return &GcsFs{ ctx: ctx, - bucket: bucket, + client: client, separator: folderSep, rawGcsObjects: make(map[string]*GcsFile), @@ -69,39 +71,65 @@ func (fs *GcsFs) ensureTrailingSeparator(s string) string { } return s } - -func (fs *GcsFs) ensureNoLeadingSeparators(s string) string { - // GCS does REALLY not like the names, that begin with a separator +func (fs *GcsFs) ensureNoLeadingSeparator(s string) string { if len(s) > 0 && strings.HasPrefix(s, fs.separator) { - log.Printf( - "WARNING: the provided path \"%s\" starts with a separator \"%s\", which is not supported by "+ - "GCloud. The separator will be automatically trimmed", - s, - fs.separator, - ) - return s[len(fs.separator):] + s = s[len(fs.separator):] } + return s } -func correctTheDot(s string) string { - // So, Afero's Glob likes to give "." as a name - that to list the "empty" dir name. - // GCS _really_ dislikes the dot and gives no entries for it - so we should rather replace the dot - // with an empty string - if s == "." { - return "" +func ensureNoPrefix(s string) string { + if len(s) > 0 && strings.HasPrefix(s, gsPrefix) { + return s[len(gsPrefix):] } return s } -func (fs *GcsFs) getObj(name string) stiface.ObjectHandle { - return fs.bucket.Object(name) +func validateName(s string) error { + if len(s) == 0 { + return ErrNoBucketInName + } + return nil +} + +// Splits provided name into bucket name and path +func (fs *GcsFs) splitName(name string) (bucketName string, path string) { + splitName := strings.Split(name, fs.separator) + + return splitName[0], strings.Join(splitName[1:], fs.separator) +} + +func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) { + bucket := fs.buckets[name] + if bucket == nil { + bucket = fs.client.Bucket(name) + _, err := bucket.Attrs(fs.ctx) + if err != nil { + return nil, err + } + } + return bucket, nil +} + +func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) { + bucketName, path := fs.splitName(name) + + bucket, err := fs.getBucket(bucketName) + if err != nil { + return nil, err + } + + return bucket.Object(path), nil } func (fs *GcsFs) Name() string { return "GcsFs" } func (fs *GcsFs) Create(name string) (*GcsFile, error) { - name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))) + name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) + if err := validateName(name); err != nil { + return nil, err + } if !fs.autoRemoveEmptyFolders { baseDir := filepath.Base(name) @@ -113,9 +141,11 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) { } } - obj := fs.getObj(name) + obj, err := fs.getObj(name) + if err != nil { + return nil, err + } w := obj.NewWriter(fs.ctx) - var err error err = w.Close() if err != nil { return nil, err @@ -127,15 +157,24 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) { } func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error { - name = fs.ensureTrailingSeparator(fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name)))) + name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name)))) + if err := validateName(name); err != nil { + return err + } - obj := fs.getObj(name) + obj, err := fs.getObj(name) + if err != nil { + return err + } w := obj.NewWriter(fs.ctx) return w.Close() } func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error { - path = fs.ensureTrailingSeparator(fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(path)))) + path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path)))) + if err := validateName(path); err != nil { + return err + } root := "" folders := strings.Split(path, fs.separator) @@ -165,13 +204,21 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile var file *GcsFile var err error - name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))) + name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) + if err = validateName(name); err != nil { + return nil, err + } - obj, found := fs.rawGcsObjects[name] + f, found := fs.rawGcsObjects[name] if found { - file = NewGcsFileFromOldFH(flag, fileMode, obj.resource) + file = NewGcsFileFromOldFH(flag, fileMode, f.resource) } else { - file = NewGcsFile(fs.ctx, fs, fs.getObj(name), flag, fileMode, name) + var obj stiface.ObjectHandle + obj, err = fs.getObj(name) + if err != nil { + return nil, err + } + file = NewGcsFile(fs.ctx, fs, obj, flag, fileMode, name) } if flag == os.O_RDONLY { @@ -211,9 +258,15 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile } func (fs *GcsFs) Remove(name string) error { - name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))) + name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) + if err := validateName(name); err != nil { + return err + } - obj := fs.getObj(name) + obj, err := fs.getObj(name) + if err != nil { + return err + } info, err := fs.Stat(name) if err != nil { return err @@ -235,7 +288,10 @@ func (fs *GcsFs) Remove(name string) error { // it's an empty folder, we can continue name = fs.ensureTrailingSeparator(name) - obj = fs.getObj(name) + obj, err = fs.getObj(name) + if err != nil { + return err + } return obj.Delete(fs.ctx) } @@ -243,7 +299,10 @@ func (fs *GcsFs) Remove(name string) error { } func (fs *GcsFs) RemoveAll(path string) error { - path = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(path))) + path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path))) + if err := validateName(path); err != nil { + return err + } pathInfo, err := fs.Stat(path) if err != nil { @@ -262,63 +321,37 @@ func (fs *GcsFs) RemoveAll(path string) error { var infos []os.FileInfo infos, err = dir.Readdir(0) for _, info := range infos { - err = fs.RemoveAll(path + fs.separator + info.Name()) + nameToRemove := fs.normSeparators(info.Name()) + err = fs.RemoveAll(path + fs.separator + nameToRemove) if err != nil { return err } } return fs.Remove(path) - - //it := fs.bucket.Objects(fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: path, Versions: false}) - //for { - // objAttrs, err := it.Next() - // if err == iterator.Done { - // break - // } - // if err != nil { - // return err - // } - // - // name := objAttrs.Name - // if name == "" { - // name = objAttrs.Prefix - // } - // - // if name == path { - // // somehow happens - // continue - // } - // if objAttrs.Name == "" && objAttrs.Prefix != "" { - // // it's a folder, let's try to remove it normally first - // err = fs.Remove(path + fs.separator + objAttrs.Name) - // if err != nil { - // if err == syscall.ENOTEMPTY { - // err = fs.RemoveAll(path + fs.separator + objAttrs.Name) - // } - // } - // if err != nil { - // return err - // } - // - // } else { - // err = fs.Remove(objAttrs.Name) - // if err != nil { - // return err - // } - // } - //} - //return nil } func (fs *GcsFs) Rename(oldName, newName string) error { - oldName = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(oldName))) - newName = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(newName))) + oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName))) + if err := validateName(oldName); err != nil { + return err + } - src := fs.bucket.Object(oldName) - dst := fs.bucket.Object(newName) + newName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(newName))) + if err := validateName(newName); err != nil { + return err + } - if _, err := dst.CopierFrom(src).Run(fs.ctx); err != nil { + src, err := fs.getObj(oldName) + if err != nil { + return err + } + dst, err := fs.getObj(newName) + if err != nil { + return err + } + + if _, err = dst.CopierFrom(src).Run(fs.ctx); err != nil { return err } delete(fs.rawGcsObjects, oldName) @@ -326,7 +359,10 @@ func (fs *GcsFs) Rename(oldName, newName string) error { } func (fs *GcsFs) Stat(name string) (os.FileInfo, error) { - name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))) + name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) + if err := validateName(name); err != nil { + return nil, err + } return newFileInfo(name, fs, defaultFileMode) } diff --git a/go.mod b/go.mod index 8c17b678..e6cf780b 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/pkg/sftp v1.10.1 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 // indirect golang.org/x/text v0.3.4 google.golang.org/api v0.40.0 )