From 1a0f20777f85a6511dc538d6a2f555f3e77ddf0a Mon Sep 17 00:00:00 2001 From: Vasily Ovchinnikov Date: Wed, 31 Mar 2021 13:55:22 +0200 Subject: [PATCH 1/6] Add GCS Fs implementation --- gcs-fake-service-account.json | 9 + gcs.go | 113 ++++++ gcs_mocks.go | 255 ++++++++++++++ gcs_test.go | 626 ++++++++++++++++++++++++++++++++++ gcsfs/errors.go | 30 ++ gcsfs/file.go | 305 +++++++++++++++++ gcsfs/file_info.go | 137 ++++++++ gcsfs/file_resource.go | 271 +++++++++++++++ gcsfs/fs.go | 344 +++++++++++++++++++ go.mod | 8 +- go.sum | 442 +++++++++++++++++++++++- 11 files changed, 2532 insertions(+), 8 deletions(-) create mode 100644 gcs-fake-service-account.json create mode 100644 gcs.go create mode 100644 gcs_mocks.go create mode 100644 gcs_test.go create mode 100644 gcsfs/errors.go create mode 100644 gcsfs/file.go create mode 100644 gcsfs/file_info.go create mode 100644 gcsfs/file_resource.go create mode 100644 gcsfs/fs.go diff --git a/gcs-fake-service-account.json b/gcs-fake-service-account.json new file mode 100644 index 00000000..95ca5ab0 --- /dev/null +++ b/gcs-fake-service-account.json @@ -0,0 +1,9 @@ +{ + "type": "service_account", + "private_key_id": "abc", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDY3E8o1NEFcjMM\nHW/5ZfFJw29/8NEqpViNjQIx95Xx5KDtJ+nWn9+OW0uqsSqKlKGhAdAo+Q6bjx2c\nuXVsXTu7XrZUY5Kltvj94DvUa1wjNXs606r/RxWTJ58bfdC+gLLxBfGnB6CwK0YQ\nxnfpjNbkUfVVzO0MQD7UP0Hl5ZcY0Puvxd/yHuONQn/rIAieTHH1pqgW+zrH/y3c\n59IGThC9PPtugI9ea8RSnVj3PWz1bX2UkCDpy9IRh9LzJLaYYX9RUd7++dULUlat\nAaXBh1U6emUDzhrIsgApjDVtimOPbmQWmX1S60mqQikRpVYZ8u+NDD+LNw+/Eovn\nxCj2Y3z1AgMBAAECggEAWDBzoqO1IvVXjBA2lqId10T6hXmN3j1ifyH+aAqK+FVl\nGjyWjDj0xWQcJ9ync7bQ6fSeTeNGzP0M6kzDU1+w6FgyZqwdmXWI2VmEizRjwk+/\n/uLQUcL7I55Dxn7KUoZs/rZPmQDxmGLoue60Gg6z3yLzVcKiDc7cnhzhdBgDc8vd\nQorNAlqGPRnm3EqKQ6VQp6fyQmCAxrr45kspRXNLddat3AMsuqImDkqGKBmF3Q1y\nxWGe81LphUiRqvqbyUlh6cdSZ8pLBpc9m0c3qWPKs9paqBIvgUPlvOZMqec6x4S6\nChbdkkTRLnbsRr0Yg/nDeEPlkhRBhasXpxpMUBgPywKBgQDs2axNkFjbU94uXvd5\nznUhDVxPFBuxyUHtsJNqW4p/ujLNimGet5E/YthCnQeC2P3Ym7c3fiz68amM6hiA\nOnW7HYPZ+jKFnefpAtjyOOs46AkftEg07T9XjwWNPt8+8l0DYawPoJgbM5iE0L2O\nx8TU1Vs4mXc+ql9F90GzI0x3VwKBgQDqZOOqWw3hTnNT07Ixqnmd3dugV9S7eW6o\nU9OoUgJB4rYTpG+yFqNqbRT8bkx37iKBMEReppqonOqGm4wtuRR6LSLlgcIU9Iwx\nyfH12UWqVmFSHsgZFqM/cK3wGev38h1WBIOx3/djKn7BdlKVh8kWyx6uC8bmV+E6\nOoK0vJD6kwKBgHAySOnROBZlqzkiKW8c+uU2VATtzJSydrWm0J4wUPJifNBa/hVW\ndcqmAzXC9xznt5AVa3wxHBOfyKaE+ig8CSsjNyNZ3vbmr0X04FoV1m91k2TeXNod\njMTobkPThaNm4eLJMN2SQJuaHGTGERWC0l3T18t+/zrDMDCPiSLX1NAvAoGBAN1T\nVLJYdjvIMxf1bm59VYcepbK7HLHFkRq6xMJMZbtG0ryraZjUzYvB4q4VjHk2UDiC\nlhx13tXWDZH7MJtABzjyg+AI7XWSEQs2cBXACos0M4Myc6lU+eL+iA+OuoUOhmrh\nqmT8YYGu76/IBWUSqWuvcpHPpwl7871i4Ga/I3qnAoGBANNkKAcMoeAbJQK7a/Rn\nwPEJB+dPgNDIaboAsh1nZhVhN5cvdvCWuEYgOGCPQLYQF0zmTLcM+sVxOYgfy8mV\nfbNgPgsP5xmu6dw2COBKdtozw0HrWSRjACd1N4yGu75+wPCcX/gQarcjRcXXZeEa\nNtBLSfcqPULqD+h7br9lEJio\n-----END PRIVATE KEY-----\n", + "client_email": "123-abc@developer.gserviceaccount.com", + "client_id": "123-abc.apps.googleusercontent.com", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "http://localhost:8080/token" +} diff --git a/gcs.go b/gcs.go new file mode 100644 index 00000000..a9787350 --- /dev/null +++ b/gcs.go @@ -0,0 +1,113 @@ +// Copyright © 2021 Vasily Ovchinnikov . +// +// The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp +// licensed under Apache License 2.0. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package afero + +import ( + "context" + "os" + "time" + + "cloud.google.com/go/storage" + "github.com/googleapis/google-cloud-go-testing/storage/stiface" + "github.com/spf13/afero/gcsfs" + "google.golang.org/api/option" +) + +type GcsFs struct { + source *gcsfs.GcsFs +} + +// Creates a GCS file system, automatically instantiating and decorating the storage client. +// You can provide additional options to be passed to the client creation, as per +// cloud.google.com/go/storage documentation +func NewGcsFS(ctx context.Context, bucketName string, opts ...option.ClientOption) (Fs, error) { + client, err := storage.NewClient(ctx, opts...) + if err != nil { + return nil, err + } + + return NewGcsFSFromClient(ctx, client, bucketName) +} + +// The same as NewGcsFS, but the files system will use the provided folder separator. +func NewGcsFSWithSeparator(ctx context.Context, bucketName, folderSeparator string, opts ...option.ClientOption) (Fs, error) { + client, err := storage.NewClient(ctx, opts...) + if err != nil { + return nil, err + } + + return NewGcsFSFromClientWithSeparator(ctx, client, bucketName, folderSeparator) +} + +// Creates a GCS file system from a given storage client +func NewGcsFSFromClient(ctx context.Context, client *storage.Client, bucketName string) (Fs, error) { + c := stiface.AdaptClient(client) + + bucket := c.Bucket(bucketName) + + return &GcsFs{gcsfs.NewGcsFs(ctx, bucket)}, nil +} + +// Same as NewGcsFSFromClient, but the file system will use the provided folder separator. +func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, bucketName, folderSeparator string) (Fs, error) { + c := stiface.AdaptClient(client) + + bucket := c.Bucket(bucketName) + + return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, bucket, folderSeparator)}, nil +} + +// Wraps gcs.GcsFs and convert some return types to afero interfaces. +func (fs *GcsFs) Name() string { + return fs.source.Name() +} +func (fs *GcsFs) Create(name string) (File, error) { + return fs.source.Create(name) +} +func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error { + return fs.source.Mkdir(name, perm) +} +func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error { + return fs.source.MkdirAll(path, perm) +} +func (fs *GcsFs) Open(name string) (File, error) { + return fs.source.Open(name) +} +func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) { + return fs.source.OpenFile(name, flag, perm) +} +func (fs *GcsFs) Remove(name string) error { + return fs.source.Remove(name) +} +func (fs *GcsFs) RemoveAll(path string) error { + return fs.source.RemoveAll(path) +} +func (fs *GcsFs) Rename(oldname, newname string) error { + return fs.source.Rename(oldname, newname) +} +func (fs *GcsFs) Stat(name string) (os.FileInfo, error) { + return fs.source.Stat(name) +} +func (fs *GcsFs) Chmod(name string, mode os.FileMode) error { + return fs.source.Chmod(name, mode) +} +func (fs *GcsFs) Chtimes(name string, atime time.Time, mtime time.Time) error { + return fs.source.Chtimes(name, atime, mtime) +} +func (fs *GcsFs) Chown(name string, uid, gid int) error { + return fs.source.Chown(name, uid, gid) +} diff --git a/gcs_mocks.go b/gcs_mocks.go new file mode 100644 index 00000000..2d6928e3 --- /dev/null +++ b/gcs_mocks.go @@ -0,0 +1,255 @@ +// Copyright © 2021 Vasily Ovchinnikov . +// +// A set of stiface-based mocks, replicating the GCS behavior, to make the tests not require any +// internet connection or real buckets. +// It is **not** a comprehensive set of mocks to test anything and everything GCS-related, rather +// a very tailored one for the current implementation - thus the tests, written with the use of +// these mocks are more of regression ones. +// If any GCS behavior changes and breaks the implementation, then it should first be adjusted by +// switching over to a real bucket - and then the mocks have to be adjusted to match the +// implementation. + +package afero + +import ( + "context" + "io" + "os" + "strings" + + "github.com/spf13/afero/gcsfs" + + "cloud.google.com/go/storage" + "github.com/googleapis/google-cloud-go-testing/storage/stiface" + "google.golang.org/api/iterator" +) + +// sets filesystem separators to the one, expected (and hard-coded) in the tests +func normSeparators(s string) string { + return strings.Replace(s, "\\", "/", -1) +} + +type clientMock struct { + stiface.Client + fs Fs +} + +func newClientMock() *clientMock { + return &clientMock{fs: NewMemMapFs()} +} + +func (m *clientMock) Bucket(name string) stiface.BucketHandle { + return &bucketMock{bucketName: name, fs: m.fs} +} + +type bucketMock struct { + stiface.BucketHandle + + bucketName string + + fs Fs +} + +func (m *bucketMock) Object(name string) stiface.ObjectHandle { + return &objectMock{name: name, fs: m.fs} +} + +func (m *bucketMock) Objects(_ context.Context, q *storage.Query) (it stiface.ObjectIterator) { + return &objectItMock{name: q.Prefix, fs: m.fs} +} + +type objectMock struct { + stiface.ObjectHandle + + name string + fs Fs +} + +func (o *objectMock) NewWriter(_ context.Context) stiface.Writer { + return &writerMock{name: o.name, fs: o.fs} +} + +func (o *objectMock) NewRangeReader(_ context.Context, offset, length int64) (stiface.Reader, error) { + if o.name == "" { + return nil, gcsfs.ErrEmptyObjectName + } + + file, err := o.fs.Open(o.name) + if err != nil { + return nil, err + } + + if offset > 0 { + _, err = file.Seek(offset, io.SeekStart) + if err != nil { + return nil, err + } + } + + res := &readerMock{file: file} + if length > -1 { + res.buf = make([]byte, length) + _, err = file.Read(res.buf) + if err != nil { + return nil, err + } + } + + return res, nil +} + +func (o *objectMock) Delete(_ context.Context) error { + if o.name == "" { + return gcsfs.ErrEmptyObjectName + } + return o.fs.Remove(o.name) +} + +func (o *objectMock) Attrs(_ context.Context) (*storage.ObjectAttrs, error) { + if o.name == "" { + return nil, gcsfs.ErrEmptyObjectName + } + + info, err := o.fs.Stat(o.name) + if err != nil { + pathError, ok := err.(*os.PathError) + if ok { + if pathError.Err == os.ErrNotExist { + return nil, storage.ErrObjectNotExist + } + } + + return nil, err + } + + res := &storage.ObjectAttrs{Name: normSeparators(o.name), Size: info.Size(), Updated: info.ModTime()} + + if info.IsDir() { + // we have to mock it here, because of FileInfo logic + return nil, gcsfs.ErrObjectDoesNotExist + } + + return res, nil +} + +type writerMock struct { + stiface.Writer + + name string + fs Fs + + file File +} + +func (w *writerMock) Write(p []byte) (n int, err error) { + if w.file == nil { + w.file, err = w.fs.Create(w.name) + if err != nil { + return 0, err + } + } + + return w.file.Write(p) +} + +func (w *writerMock) Close() error { + if w.file == nil { + var err error + if strings.HasSuffix(w.name, "/") { + err = w.fs.Mkdir(w.name, 0755) + if err != nil { + return err + } + } else { + _, err = w.Write([]byte{}) + if err != nil { + return err + } + } + } + if w.file != nil { + return w.file.Close() + } + return nil +} + +type readerMock struct { + stiface.Reader + + file File + + buf []byte +} + +func (r *readerMock) Remain() int64 { + return 0 +} + +func (r *readerMock) Read(p []byte) (int, error) { + if r.buf != nil { + copy(p, r.buf) + return len(r.buf), nil + } + return r.file.Read(p) +} + +type objectItMock struct { + stiface.ObjectIterator + + name string + fs Fs + + dir File + infos []*storage.ObjectAttrs +} + +func (it *objectItMock) Next() (*storage.ObjectAttrs, error) { + var err error + if it.dir == nil { + it.dir, err = it.fs.Open(it.name) + if err != nil { + return nil, err + } + + var isDir bool + isDir, err = IsDir(it.fs, it.name) + if err != nil { + return nil, err + } + + it.infos = []*storage.ObjectAttrs{} + + if !isDir { + var info os.FileInfo + info, err = it.dir.Stat() + if err != nil { + return nil, err + } + it.infos = append(it.infos, &storage.ObjectAttrs{Name: normSeparators(info.Name()), Size: info.Size(), Updated: info.ModTime()}) + } else { + var fInfos []os.FileInfo + fInfos, err = it.dir.Readdir(0) + if err != nil { + return nil, err + } + if it.name != "" { + it.infos = append(it.infos, &storage.ObjectAttrs{ + Prefix: normSeparators(it.name) + "/", + }) + } + + for _, info := range fInfos { + it.infos = append(it.infos, &storage.ObjectAttrs{Name: normSeparators(info.Name()), Size: info.Size(), Updated: info.ModTime()}) + } + } + } + + if len(it.infos) == 0 { + return nil, iterator.Done + } + + res := it.infos[0] + it.infos = it.infos[1:] + + return res, err +} diff --git a/gcs_test.go b/gcs_test.go new file mode 100644 index 00000000..3b701e1f --- /dev/null +++ b/gcs_test.go @@ -0,0 +1,626 @@ +// Copyright © 2021 Vasily Ovchinnikov . +// +// Most of the tests are "derived" from the Afero's own tarfs implementation. +// Write-oriented tests and/or checks have been added on top of that + +package afero + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "syscall" + "testing" + + "github.com/spf13/afero/gcsfs" + + "cloud.google.com/go/storage" + "github.com/googleapis/google-cloud-go-testing/storage/stiface" +) + +const ( + testBytes = 8 + dirSize = 42 +) + +var files = []struct { + name string + exists bool + isdir bool + size int64 + content string + offset int64 + contentAtOffset string +}{ + {"", true, true, dirSize, "", 0, ""}, // this is NOT a valid path for GCS, so we do some magic here + {"sub", true, true, dirSize, "", 0, ""}, + {"sub/testDir2", true, true, dirSize, "", 0, ""}, + {"sub/testDir2/testFile", true, false, 8 * 1024, "c", 4 * 1024, "d"}, + {"testFile", true, false, 12 * 1024, "a", 7 * 1024, "b"}, + {"testDir1/testFile", true, false, 3 * 512, "b", 512, "c"}, + + {"nonExisting", false, false, dirSize, "", 0, ""}, +} + +var dirs = []struct { + name string + children []string +}{ + {"", []string{"sub", "testDir1", "testFile"}}, + {"sub", []string{"testDir2"}}, + {"sub/testDir2", []string{"testFile"}}, + {"testDir1", []string{"testFile"}}, +} + +var gcsAfs *Afero + +func TestMain(m *testing.M) { + ctx := context.Background() + var err error + + // Check if GOOGLE_APPLICATION_CREDENTIALS are present. If not, then a fake service account + // would be used: https://github.com/google/oauth2l/blob/master/integration/fixtures/fake-service-account.json + if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { + var fakeCredentialsAbsPath string + fakeCredentialsAbsPath, err = filepath.Abs("gcs-fake-service-account.json") + if err != nil { + fmt.Print(err) + os.Exit(1) + } + + err = os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", fakeCredentialsAbsPath) + if err != nil { + fmt.Print(err) + os.Exit(1) + } + + // reset it after the run + defer func() { + err = os.Remove("GOOGLE_APPLICATION_CREDENTIALS") + if err != nil { + // it's worth printing it out explicitly, since it might have implications further down the road + fmt.Print("failed to clear fake GOOGLE_APPLICATION_CREDENTIALS", err) + } + }() + } + + var c *storage.Client + c, err = storage.NewClient(ctx) + if err != nil { + fmt.Print(err) + os.Exit(1) + } + client := stiface.AdaptClient(c) + + // This block is mocking the client for the sake of isolated testing + mockClient := newClientMock() + mockClient.Client = client + + bucket := mockClient.Bucket("a-test-bucket") + + // If you want to run the test suite on a LIVE bucket, comment the previous + // block and uncomment the line below and put your bucket name there. + // Keep in mind, that GCS will likely rate limit you, so it would be impossible + // to run the entire suite at once, only test by test. + //bucket := client.Bucket("a-test-bucket") + + gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, bucket)}} + + // defer here to assure our Env cleanup happens, if the mock was used + defer os.Exit(m.Run()) +} + +func createFiles(t *testing.T) { + t.Helper() + var err error + + // the files have to be created first + for _, f := range files { + if !f.isdir && f.exists { + var freshFile File + freshFile, err = gcsAfs.Create(f.name) + if err != nil { + t.Fatalf("failed to create a file \"%s\": %s", f.name, err) + } + + var written int + var totalWritten int64 + for totalWritten < f.size { + if totalWritten < f.offset { + writeBuf := []byte(strings.Repeat(f.content, int(f.offset))) + written, err = freshFile.WriteAt(writeBuf, totalWritten) + } else { + writeBuf := []byte(strings.Repeat(f.contentAtOffset, int(f.size-f.offset))) + written, err = freshFile.WriteAt(writeBuf, totalWritten) + } + if err != nil { + t.Fatalf("failed to write a file \"%s\": %s", f.name, err) + } + + totalWritten += int64(written) + } + + err = freshFile.Close() + if err != nil { + t.Fatalf("failed to close a file \"%s\": %s", f.name, err) + } + } + } +} + +func removeFiles(t *testing.T) { + t.Helper() + var err error + + // the files have to be created first + for _, f := range files { + if !f.isdir && f.exists { + err = gcsAfs.Remove(f.name) + if err != nil && err == syscall.ENOENT { + t.Errorf("failed to remove file \"%s\": %s", f.name, err) + } + } + } +} + +func TestFsOpen(t *testing.T) { + createFiles(t) + defer removeFiles(t) + + for _, f := range files { + file, err := gcsAfs.Open(f.name) + if (err == nil) != f.exists { + t.Errorf("%v exists = %v, but got err = %v", f.name, f.exists, err) + } + + if !f.exists { + continue + } + if err != nil { + t.Fatalf("%v: %v", f.name, err) + } + + if file.Name() != filepath.FromSlash(f.name) { + t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(f.name)) + } + + s, err := file.Stat() + if err != nil { + t.Fatalf("stat %v: got error '%v'", file.Name(), err) + } + + if isdir := s.IsDir(); isdir != f.isdir { + t.Errorf("%v directory, got: %v, expected: %v", file.Name(), isdir, f.isdir) + } + + if size := s.Size(); size != f.size { + t.Errorf("%v size, got: %v, expected: %v", file.Name(), size, f.size) + } + } +} + +func TestRead(t *testing.T) { + createFiles(t) + defer removeFiles(t) + + for _, f := range files { + if !f.exists { + continue + } + + file, err := gcsAfs.Open(f.name) + if err != nil { + t.Fatalf("opening %v: %v", f.name, err) + } + + buf := make([]byte, 8) + n, err := file.Read(buf) + if err != nil { + if f.isdir && (err != syscall.EISDIR) { + t.Errorf("%v got error %v, expected EISDIR", f.name, err) + } else if !f.isdir { + t.Errorf("%v: %v", f.name, err) + } + } else if n != 8 { + t.Errorf("%v: got %d read bytes, expected 8", f.name, n) + } else if string(buf) != strings.Repeat(f.content, testBytes) { + t.Errorf("%v: got <%s>, expected <%s>", f.name, f.content, string(buf)) + } + + } +} + +func TestGcsReadAt(t *testing.T) { + createFiles(t) + defer removeFiles(t) + + for _, f := range files { + if !f.exists { + continue + } + + file, err := gcsAfs.Open(f.name) + if err != nil { + t.Fatalf("opening %v: %v", f.name, err) + } + + buf := make([]byte, testBytes) + n, err := file.ReadAt(buf, f.offset-testBytes/2) + if err != nil { + if f.isdir && (err != syscall.EISDIR) { + t.Errorf("%v got error %v, expected EISDIR", f.name, err) + } else if !f.isdir { + t.Errorf("%v: %v", f.name, err) + } + } else if n != 8 { + t.Errorf("%v: got %d read bytes, expected 8", f.name, n) + } else if string(buf) != strings.Repeat(f.content, testBytes/2)+strings.Repeat(f.contentAtOffset, testBytes/2) { + t.Errorf("%v: got <%s>, expected <%s>", f.name, f.contentAtOffset, string(buf)) + } + + } +} + +func TestGcsSeek(t *testing.T) { + createFiles(t) + defer removeFiles(t) + + for _, f := range files { + if !f.exists { + continue + } + + file, err := gcsAfs.Open(f.name) + if err != nil { + t.Fatalf("opening %v: %v", f.name, err) + } + + var tests = []struct { + offIn int64 + whence int + offOut int64 + }{ + {0, io.SeekStart, 0}, + {10, io.SeekStart, 10}, + {1, io.SeekCurrent, 11}, + {10, io.SeekCurrent, 21}, + {0, io.SeekEnd, f.size}, + {-1, io.SeekEnd, f.size - 1}, + } + + for _, s := range tests { + n, err := file.Seek(s.offIn, s.whence) + if err != nil { + if f.isdir && err == syscall.EISDIR { + continue + } + + t.Errorf("%v: %v", f.name, err) + } + + if n != s.offOut { + t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offIn, s.whence, n, s.offOut) + } + } + + } +} + +func TestName(t *testing.T) { + createFiles(t) + defer removeFiles(t) + + for _, f := range files { + if !f.exists { + continue + } + + file, err := gcsAfs.Open(f.name) + if err != nil { + t.Fatalf("opening %v: %v", f.name, err) + } + + n := file.Name() + if n != filepath.FromSlash(f.name) { + t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(f.name)) + } + + } +} + +func TestClose(t *testing.T) { + createFiles(t) + defer removeFiles(t) + + for _, f := range files { + if !f.exists { + continue + } + + file, err := gcsAfs.Open(f.name) + if err != nil { + t.Fatalf("opening %v: %v", f.name, err) + } + + err = file.Close() + if err != nil { + t.Errorf("%v: %v", f.name, err) + } + + err = file.Close() + if err == nil { + t.Errorf("%v: closing twice should return an error", f.name) + } + + buf := make([]byte, 8) + n, err := file.Read(buf) + if n != 0 || err == nil { + t.Errorf("%v: could read from a closed file", f.name) + } + + n, err = file.ReadAt(buf, 256) + if n != 0 || err == nil { + t.Errorf("%v: could readAt from a closed file", f.name) + } + + off, err := file.Seek(0, io.SeekStart) + if off != 0 || err == nil { + t.Errorf("%v: could seek from a closed file", f.name) + } + } +} + +func TestGcsOpenFile(t *testing.T) { + createFiles(t) + defer removeFiles(t) + + for _, f := range files { + file, err := gcsAfs.OpenFile(f.name, os.O_RDONLY, 0400) + if !f.exists { + if !errors.Is(err, syscall.ENOENT) { + t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT) + } + + continue + } + + if err != nil { + t.Fatalf("%v: %v", f.name, err) + } + + err = file.Close() + if err != nil { + t.Fatalf("failed to close a file \"%s\": %s", f.name, err) + } + + file, err = gcsAfs.OpenFile(f.name, os.O_CREATE, 0600) + if !errors.Is(err, syscall.EPERM) { + t.Errorf("%v: open for write: got %v, expected %v", f.name, err, syscall.EPERM) + } + + } +} + +func TestFsStat(t *testing.T) { + createFiles(t) + defer removeFiles(t) + + for _, f := range files { + fi, err := gcsAfs.Stat(f.name) + if !f.exists { + if !errors.Is(err, syscall.ENOENT) { + t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT) + } + + continue + } + + if err != nil { + t.Fatalf("stat %v: got error '%v'", f.name, err) + } + + if isdir := fi.IsDir(); isdir != f.isdir { + t.Errorf("%v directory, got: %v, expected: %v", f.name, isdir, f.isdir) + } + + if size := fi.Size(); size != f.size { + t.Errorf("%v size, got: %v, expected: %v", f.name, size, f.size) + } + } +} + +func TestGcsReaddir(t *testing.T) { + createFiles(t) + defer removeFiles(t) + + for _, d := range dirs { + dir, err := gcsAfs.Open(d.name) + if err != nil { + t.Fatal(err) + } + + fi, err := dir.Readdir(0) + if err != nil { + t.Fatal(err) + } + var names []string + for _, f := range fi { + names = append(names, f.Name()) + } + + if !reflect.DeepEqual(names, d.children) { + t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children) + } + + fi, err = dir.Readdir(1) + if err != nil { + t.Fatal(err) + } + + names = []string{} + for _, f := range fi { + names = append(names, f.Name()) + } + + if !reflect.DeepEqual(names, d.children[0:1]) { + t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1]) + } + } + + dir, err := gcsAfs.Open("testFile") + if err != nil { + t.Fatal(err) + } + + _, err = dir.Readdir(-1) + if err != syscall.ENOTDIR { + t.Fatal("Expected error") + } +} + +func TestGcsReaddirnames(t *testing.T) { + createFiles(t) + defer removeFiles(t) + + for _, d := range dirs { + dir, err := gcsAfs.Open(d.name) + if err != nil { + t.Fatal(err) + } + + names, err := dir.Readdirnames(0) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(names, d.children) { + t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children) + } + + names, err = dir.Readdirnames(1) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(names, d.children[0:1]) { + t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1]) + } + } + + dir, err := gcsAfs.Open("testFile") + if err != nil { + t.Fatal(err) + } + + _, err = dir.Readdir(-1) + if err != syscall.ENOTDIR { + t.Fatal("Expected error") + } +} + +func TestGcsGlob(t *testing.T) { + createFiles(t) + defer removeFiles(t) + + for _, s := range []struct { + glob string + entries []string + }{ + {filepath.FromSlash("/*"), []string{filepath.FromSlash("/sub"), filepath.FromSlash("/testDir1"), filepath.FromSlash("/testFile")}}, + {filepath.FromSlash("*"), []string{filepath.FromSlash("sub"), filepath.FromSlash("testDir1"), filepath.FromSlash("testFile")}}, + {filepath.FromSlash("/sub/*"), []string{filepath.FromSlash("/sub/testDir2")}}, + {filepath.FromSlash("/sub/testDir2/*"), []string{filepath.FromSlash("/sub/testDir2/testFile")}}, + {filepath.FromSlash("/testDir1/*"), []string{filepath.FromSlash("/testDir1/testFile")}}, + {filepath.FromSlash("sub/*"), []string{filepath.FromSlash("sub/testDir2")}}, + {filepath.FromSlash("sub/testDir2/*"), []string{filepath.FromSlash("sub/testDir2/testFile")}}, + {filepath.FromSlash("testDir1/*"), []string{filepath.FromSlash("testDir1/testFile")}}, + } { + entries, err := Glob(gcsAfs.Fs, s.glob) + if err != nil { + t.Error(err) + } + if reflect.DeepEqual(entries, s.entries) { + t.Logf("glob: %s: glob ok", s.glob) + } else { + t.Errorf("glob: %s: got %#v, expected %#v", s.glob, entries, s.entries) + } + } +} + +func TestMkdir(t *testing.T) { + dirName := "/a-test-dir" + var err error + + err = gcsAfs.Mkdir(dirName, 0755) + if err != nil { + t.Fatal("failed to create a folder with error", err) + } + + info, err := gcsAfs.Stat(dirName) + if err != nil { + t.Fatal("failed to get info", err) + } + if !info.IsDir() { + t.Fatalf("%s: not a dir", dirName) + } + if !info.Mode().IsDir() { + t.Errorf("%s: mode is not directory", dirName) + } + + if info.Mode() != os.ModeDir|0755 { + t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode()) + } + + err = gcsAfs.Remove(dirName) + if err != nil { + t.Fatalf("could not delete the folder %s after the test with error: %s", dirName, err) + } +} + +func TestMkdirAll(t *testing.T) { + err := gcsAfs.MkdirAll("/a/b/c", 0755) + if err != nil { + t.Fatal(err) + } + + info, err := gcsAfs.Stat("/a") + if err != nil { + t.Fatal(err) + } + if !info.Mode().IsDir() { + t.Error("/a: mode is not directory") + } + if info.Mode() != os.ModeDir|0755 { + t.Errorf("/a: wrong permissions, expected drwxr-xr-x, got %s", info.Mode()) + } + info, err = gcsAfs.Stat("/a/b") + if err != nil { + t.Fatal(err) + } + if !info.Mode().IsDir() { + t.Error("/a/b: mode is not directory") + } + if info.Mode() != os.ModeDir|0755 { + t.Errorf("/a/b: wrong permissions, expected drwxr-xr-x, got %s", info.Mode()) + } + info, err = gcsAfs.Stat("/a/b/c") + if err != nil { + t.Fatal(err) + } + if !info.Mode().IsDir() { + t.Error("/a/b/c: mode is not directory") + } + if info.Mode() != os.ModeDir|0755 { + t.Errorf("/a/b/c: wrong permissions, expected drwxr-xr-x, got %s", info.Mode()) + } + + err = gcsAfs.RemoveAll("/a") + if err != nil { + t.Fatalf("failed to remove the folder /a with error: %s", err) + } +} diff --git a/gcsfs/errors.go b/gcsfs/errors.go new file mode 100644 index 00000000..7468ad5b --- /dev/null +++ b/gcsfs/errors.go @@ -0,0 +1,30 @@ +// Copyright © 2021 Vasily Ovchinnikov . +// +// The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp +// licensed under Apache License 2.0. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsfs + +import ( + "errors" + "syscall" +) + +var ( + ErrFileClosed = errors.New("file is closed") + ErrOutOfRange = errors.New("out of range") + ErrObjectDoesNotExist = errors.New("storage: object doesn't exist") + ErrEmptyObjectName = errors.New("storage: object name is empty") + ErrFileNotFound = syscall.ENOENT +) diff --git a/gcsfs/file.go b/gcsfs/file.go new file mode 100644 index 00000000..33364a4a --- /dev/null +++ b/gcsfs/file.go @@ -0,0 +1,305 @@ +// Copyright © 2021 Vasily Ovchinnikov . +// +// The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp +// licensed under Apache License 2.0. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsfs + +import ( + "context" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sort" + "syscall" + + "github.com/googleapis/google-cloud-go-testing/storage/stiface" + + "cloud.google.com/go/storage" + + "google.golang.org/api/iterator" +) + +// GcsFs is the Afero version adapted for GCS +type GcsFile struct { + openFlags int + fhOffset int64 //File handle specific offset + closed bool + ReadDirIt stiface.ObjectIterator + resource *gcsFileResource +} + +func NewGcsFile( + ctx context.Context, + fs *GcsFs, + obj stiface.ObjectHandle, + openFlags int, + // Unused: there is no use to the file mode in GCloud just yet - but we keep it here, just in case we need it + fileMode os.FileMode, + name string, +) *GcsFile { + return &GcsFile{ + openFlags: openFlags, + fhOffset: 0, + closed: false, + ReadDirIt: nil, + resource: &gcsFileResource{ + ctx: ctx, + fs: fs, + + obj: obj, + name: name, + fileMode: fileMode, + + currentGcsSize: 0, + + offset: 0, + reader: nil, + writer: nil, + }, + } +} + +func NewGcsFileFromOldFH( + openFlags int, + fileMode os.FileMode, + oldFile *gcsFileResource, +) *GcsFile { + res := &GcsFile{ + openFlags: openFlags, + fhOffset: 0, + closed: false, + ReadDirIt: nil, + + resource: oldFile, + } + res.resource.fileMode = fileMode + + return res +} + +func (o *GcsFile) Close() error { + if o.closed { + // the afero spec expects the call to Close on a closed file to return an error + return ErrFileClosed + } + o.closed = true + return o.resource.Close() +} + +func (o *GcsFile) Seek(newOffset int64, whence int) (int64, error) { + if o.closed { + return 0, ErrFileClosed + } + + //Since this is an expensive operation; let's make sure we need it + if (whence == 0 && newOffset == o.fhOffset) || (whence == 1 && newOffset == 0) { + return o.fhOffset, nil + } + log.Printf("WARNING: Seek beavhior triggered, highly inefficent. Offset before seek is at %d\n", o.fhOffset) + + //Fore the reader/writers to be reopened (at correct offset) + err := o.Sync() + if err != nil { + return 0, err + } + stat, err := o.Stat() + if err != nil { + return 0, nil + } + + switch whence { + case 0: + o.fhOffset = newOffset + case 1: + o.fhOffset += newOffset + case 2: + o.fhOffset = stat.Size() + newOffset + } + return o.fhOffset, nil +} + +func (o *GcsFile) Read(p []byte) (n int, err error) { + return o.ReadAt(p, o.fhOffset) +} + +func (o *GcsFile) ReadAt(p []byte, off int64) (n int, err error) { + if o.closed { + return 0, ErrFileClosed + } + + read, err := o.resource.ReadAt(p, off) + o.fhOffset += int64(read) + return read, err +} + +func (o *GcsFile) Write(p []byte) (n int, err error) { + return o.WriteAt(p, o.fhOffset) +} + +func (o *GcsFile) WriteAt(b []byte, off int64) (n int, err error) { + if o.closed { + return 0, ErrFileClosed + } + + if o.openFlags&os.O_RDONLY != 0 { + return 0, fmt.Errorf("file is opend as read only") + } + + _, err = o.resource.obj.Attrs(o.resource.ctx) + if err != nil { + if err == storage.ErrObjectNotExist { + if o.openFlags&os.O_CREATE == 0 { + return 0, ErrFileNotFound + } + } else { + return 0, fmt.Errorf("error getting file attributes: %v", err) + } + } + + written, err := o.resource.WriteAt(b, off) + o.fhOffset += int64(written) + return written, err +} + +func (o *GcsFile) Name() string { + return filepath.FromSlash(o.resource.name) +} + +func (o *GcsFile) readdirImpl(count int) ([]*FileInfo, error) { + err := o.Sync() + if err != nil { + return nil, err + } + + var ownInfo os.FileInfo + ownInfo, err = o.Stat() + if err != nil { + return nil, err + } + + if !ownInfo.IsDir() { + return nil, syscall.ENOTDIR + } + + path := o.resource.fs.ensureTrailingSeparator(o.Name()) + if o.ReadDirIt == nil { + //log.Printf("Querying path : %s\n", path) + o.ReadDirIt = o.resource.fs.bucket.Objects( + o.resource.ctx, &storage.Query{Delimiter: o.resource.fs.separator, Prefix: path, Versions: false}) + } + var res []*FileInfo + for { + object, err := o.ReadDirIt.Next() + if err == iterator.Done { + // reset the iterator + o.ReadDirIt = nil + + if len(res) > 0 || count <= 0 { + return res, nil + } + + return res, io.EOF + } + if err != nil { + return res, err + } + + tmp := newFileInfoFromAttrs(object, o.resource.fileMode) + + if tmp.Name() == "" { + // neither object.Name, not object.Prefix were present - so let's skip this unknown thing + continue + } + + if object.Name == "" && object.Prefix == "" { + continue + } + + if tmp.Name() == ownInfo.Name() { + // Hmmm + continue + } + + res = append(res, tmp) + + // This would interrupt the iteration, once we reach the count. + // But it would then have files coming before folders - that's not what we want to have exactly, + // since it makes the results unpredictable. Hence, we iterate all the objects and then do + // the cut-off in a higher level method + //if count > 0 && len(res) >= count { + // break + //} + } + //return res, nil +} + +func (o *GcsFile) Readdir(count int) ([]os.FileInfo, error) { + fi, err := o.readdirImpl(count) + if len(fi) > 0 { + sort.Sort(ByName(fi)) + } + + if count > 0 { + fi = fi[:count] + } + + var res []os.FileInfo + for _, f := range fi { + res = append(res, f) + } + return res, err +} + +func (o *GcsFile) Readdirnames(n int) ([]string, error) { + fi, err := o.Readdir(n) + if err != nil && err != io.EOF { + return nil, err + } + names := make([]string, len(fi)) + + for i, f := range fi { + names[i] = f.Name() + } + return names, err +} + +func (o *GcsFile) Stat() (os.FileInfo, error) { + err := o.Sync() + if err != nil { + return nil, err + } + + return newFileInfo(o.Name(), o.resource.fs, o.resource.fileMode) +} + +func (o *GcsFile) Sync() error { + return o.resource.maybeCloseIo() +} + +func (o *GcsFile) Truncate(wantedSize int64) error { + if o.closed { + return ErrFileClosed + } + if o.openFlags == os.O_RDONLY { + return fmt.Errorf("file was opened as read only") + } + return o.resource.Truncate(wantedSize) +} + +func (o *GcsFile) WriteString(s string) (ret int, err error) { + return o.Write([]byte(s)) +} diff --git a/gcsfs/file_info.go b/gcsfs/file_info.go new file mode 100644 index 00000000..b14a5f6a --- /dev/null +++ b/gcsfs/file_info.go @@ -0,0 +1,137 @@ +// Copyright © 2021 Vasily Ovchinnikov . +// +// The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp +// licensed under Apache License 2.0. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsfs + +import ( + "os" + "path/filepath" + "strings" + "time" + + "cloud.google.com/go/storage" +) + +const ( + folderSize = 42 +) + +type FileInfo struct { + name string + size int64 + updated time.Time + isDir bool + fileMode os.FileMode +} + +func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error) { + res := &FileInfo{ + name: name, + size: folderSize, + updated: time.Time{}, + isDir: false, + fileMode: fileMode, + } + + obj := fs.getObj(name) + + objAttrs, err := obj.Attrs(fs.ctx) + if err != nil { + if err.Error() == ErrEmptyObjectName.Error() { + // It's a root folder here, we return right away + res.name = fs.ensureTrailingSeparator(res.name) + res.isDir = true + return res, nil + } else if err.Error() == ErrObjectDoesNotExist.Error() { + // Folders do not actually "exist" in GCloud, so we have to check, if something exists with + // such a prefix + it := fs.bucket.Objects( + fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: name, Versions: false}) + if _, err = it.Next(); err == nil { + res.name = fs.ensureTrailingSeparator(res.name) + res.isDir = true + return res, nil + } + + return nil, ErrFileNotFound + } + return nil, err + } + + res.size = objAttrs.Size + res.updated = objAttrs.Updated + + return res, nil +} + +func newFileInfoFromAttrs(objAttrs *storage.ObjectAttrs, fileMode os.FileMode) *FileInfo { + res := &FileInfo{ + name: objAttrs.Name, + size: objAttrs.Size, + updated: objAttrs.Updated, + isDir: false, + fileMode: fileMode, + } + + if res.name == "" { + if objAttrs.Prefix != "" { + // It's a virtual folder! It does not have a name, but prefix - this is how GCS API + // deals with them at the moment + res.name = objAttrs.Prefix + res.size = folderSize + res.isDir = true + } + } + + return res +} + +func (fi *FileInfo) Name() string { + return filepath.Base(fi.name) +} + +func (fi *FileInfo) Size() int64 { + return fi.size +} +func (fi *FileInfo) Mode() os.FileMode { + if fi.IsDir() { + return os.ModeDir | fi.fileMode + } + return fi.fileMode +} + +func (fi *FileInfo) ModTime() time.Time { + return fi.updated +} + +func (fi *FileInfo) IsDir() bool { + return fi.isDir +} + +func (fi *FileInfo) Sys() interface{} { + return nil +} + +type ByName []*FileInfo + +func (a ByName) Len() int { return len(a) } +func (a ByName) Swap(i, j int) { + a[i].name, a[j].name = a[j].name, a[i].name + a[i].size, a[j].size = a[j].size, a[i].size + a[i].updated, a[j].updated = a[j].updated, a[i].updated + a[i].isDir, a[j].isDir = a[j].isDir, a[i].isDir +} +func (a ByName) Less(i, j int) bool { return strings.Compare(a[i].Name(), a[j].Name()) == -1 } diff --git a/gcsfs/file_resource.go b/gcsfs/file_resource.go new file mode 100644 index 00000000..b25d0e86 --- /dev/null +++ b/gcsfs/file_resource.go @@ -0,0 +1,271 @@ +// Copyright © 2021 Vasily Ovchinnikov . +// +// The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp +// licensed under Apache License 2.0. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsfs + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "syscall" + + "github.com/googleapis/google-cloud-go-testing/storage/stiface" +) + +const ( + maxWriteSize = 10000 +) + +// gcsFileResource represents a singleton version of each GCS object; +// Google cloud storage allows users to open multiple writers(!) to the same +// underlying resource, once the write is closed the written stream is commented. We are doing +// some magic where we read and and write to the same file which requires synchronization +// of the underlying resource. + +type gcsFileResource struct { + ctx context.Context + + fs *GcsFs + + obj stiface.ObjectHandle + name string + fileMode os.FileMode + + currentGcsSize int64 + offset int64 + reader io.ReadCloser + writer io.WriteCloser + + closed bool +} + +func (o *gcsFileResource) Close() error { + o.closed = true + // TODO rawGcsObjectsMap ? + return o.maybeCloseIo() +} + +func (o *gcsFileResource) maybeCloseIo() error { + if err := o.maybeCloseReader(); err != nil { + return fmt.Errorf("error closing reader: %v", err) + } + if err := o.maybeCloseWriter(); err != nil { + return fmt.Errorf("error closing writer: %v", err) + } + + return nil +} + +func (o *gcsFileResource) maybeCloseReader() error { + if o.reader == nil { + return nil + } + if err := o.reader.Close(); err != nil { + return err + } + o.reader = nil + return nil +} + +func (o *gcsFileResource) maybeCloseWriter() error { + if o.writer == nil { + return nil + } + + // In cases of partial writes (e.g. to the middle of a file stream), we need to + // append any remaining data from the original file before we close the reader (and + // commit the results.) + // For small writes it can be more efficient + // to keep the original reader but that is for another iteration + if o.currentGcsSize > o.offset { + currentFile, err := o.obj.NewRangeReader(o.ctx, o.offset, -1) + if err != nil { + return fmt.Errorf( + "couldn't simulate a partial write; the closing (and thus"+ + " the whole file write) is NOT commited to GCS. %v", err) + } + if currentFile != nil && currentFile.Remain() > 0 { + if _, err := io.Copy(o.writer, currentFile); err != nil { + return fmt.Errorf("error writing: %v", err) + } + } + } + + if err := o.writer.Close(); err != nil { + return err + } + o.writer = nil + return nil +} + +func (o *gcsFileResource) ReadAt(p []byte, off int64) (n int, err error) { + if cap(p) == 0 { + return 0, nil + } + + // Assume that if the reader is open; it is at the correct offset + // a good performance assumption that we must ensure holds + if off == o.offset && o.reader != nil { + n, err = o.reader.Read(p) + o.offset += int64(n) + return n, err + } + + // we have to check, whether it's a folder; the folder must not have an open readers, or writers though, + // so this check should not be invoked excessively and cause too much of a performance drop + if o.reader == nil && o.writer == nil { + var info *FileInfo + info, err = newFileInfo(o.name, o.fs, o.fileMode) + if err != nil { + return 0, err + } + + if info.IsDir() { + // trying to read a directory must return this + return 0, syscall.EISDIR + } + } + + // If any writers have written anything; commit it first so we can read it back. + if err = o.maybeCloseIo(); err != nil { + return 0, err + } + + //Then read at the correct offset. + r, err := o.obj.NewRangeReader(o.ctx, off, -1) + if err != nil { + return 0, err + } + o.reader = r + o.offset = off + + read, err := o.reader.Read(p) + o.offset += int64(read) + return read, err +} + +func (o *gcsFileResource) WriteAt(b []byte, off int64) (n int, err error) { + //If the writer is opened and at the correct offset we're good! + if off == o.offset && o.writer != nil { + n, err = o.writer.Write(b) + o.offset += int64(n) + return n, err + } + + // Ensure readers must be re-opened and that if a writer is active at another + // offset it is first committed before we do a "seek" below + if err = o.maybeCloseIo(); err != nil { + return 0, err + } + + w := o.obj.NewWriter(o.ctx) + // TRIGGER WARNING: This can seem like a hack but it works thanks + // to GCS strong consistency. We will open and write to the same file; First when the + // writer is closed will the content get committed to GCS. + // The general idea is this: + // Objectv1[:offset] -> Objectv2 + // newData1 -> Objectv2 + // Objectv1[offset+len(newData1):] -> Objectv2 + // Objectv2.Close + // + // It will however require a download and upload of the original file but it + // can't be avoided if we should support seek-write-operations on GCS. + objAttrs, err := o.obj.Attrs(o.ctx) + if err != nil { + if off > 0 { + return 0, err // WriteAt to a non existing file + } + + o.currentGcsSize = 0 + } else { + o.currentGcsSize = objAttrs.Size + } + + if off > o.currentGcsSize { + return 0, ErrOutOfRange + } + + if off > 0 { + var r stiface.Reader + r, err = o.obj.NewReader(o.ctx) + if err != nil { + return 0, err + } + if _, err = io.CopyN(w, r, off); err != nil { + return 0, err + } + if err = r.Close(); err != nil { + return 0, err + } + } + + o.writer = w + o.offset = off + + written, err := o.writer.Write(b) + + o.offset += int64(written) + return written, err +} + +func min(x, y int) int { + if x < y { + return x + } + return y +} + +func (o *gcsFileResource) Truncate(wantedSize int64) error { + if wantedSize < 0 { + return ErrOutOfRange + } + + if err := o.maybeCloseIo(); err != nil { + return err + } + + r, err := o.obj.NewRangeReader(o.ctx, 0, wantedSize) + if err != nil { + return err + } + + w := o.obj.NewWriter(o.ctx) + written, err := io.Copy(w, r) + if err != nil { + return err + } + + for written < wantedSize { + //Bulk up padding writes + paddingBytes := bytes.Repeat([]byte(" "), min(maxWriteSize, int(wantedSize-written))) + + n := 0 + if n, err = w.Write(paddingBytes); err != nil { + return err + } + + written += int64(n) + } + if err = r.Close(); err != nil { + return fmt.Errorf("error closing reader: %v", err) + } + if err = w.Close(); err != nil { + return fmt.Errorf("error closing writer: %v", err) + } + return nil +} diff --git a/gcsfs/fs.go b/gcsfs/fs.go new file mode 100644 index 00000000..79bfa3b7 --- /dev/null +++ b/gcsfs/fs.go @@ -0,0 +1,344 @@ +// Copyright © 2021 Vasily Ovchinnikov . +// +// The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp +// licensed under Apache License 2.0. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcsfs + +import ( + "context" + "errors" + "log" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/googleapis/google-cloud-go-testing/storage/stiface" +) + +const ( + defaultFileMode = 0755 +) + +// GcsFs is a Fs implementation that uses functions provided by google cloud storage +type GcsFs struct { + ctx context.Context + bucket stiface.BucketHandle + separator string + rawGcsObjects map[string]*GcsFile + + autoRemoveEmptyFolders bool //trigger for creating "virtual folders" (not required by GCSs) +} + +func NewGcsFs(ctx context.Context, bucket stiface.BucketHandle) *GcsFs { + return NewGcsFsWithSeparator(ctx, bucket, "/") +} + +func NewGcsFsWithSeparator(ctx context.Context, bucket stiface.BucketHandle, folderSep string) *GcsFs { + return &GcsFs{ + ctx: ctx, + bucket: bucket, + separator: folderSep, + rawGcsObjects: make(map[string]*GcsFile), + + autoRemoveEmptyFolders: true, + } +} + +// normSeparators will normalize all "\\" and "/" to the provided separator +func (fs *GcsFs) normSeparators(s string) string { + return strings.Replace(strings.Replace(s, "\\", fs.separator, -1), "/", fs.separator, -1) +} + +func (fs *GcsFs) ensureTrailingSeparator(s string) string { + if len(s) > 0 && !strings.HasSuffix(s, fs.separator) { + return s + fs.separator + } + return s +} + +func (fs *GcsFs) ensureNoLeadingSeparators(s string) string { + // GCS does REALLY not like the names, that begin with a separator + if len(s) > 0 && strings.HasPrefix(s, fs.separator) { + log.Printf( + "WARNING: the provided path \"%s\" starts with a separator \"%s\", which is not supported by "+ + "GCloud. The separator will be automatically trimmed", + s, + fs.separator, + ) + return s[len(fs.separator):] + } + return s +} + +func correctTheDot(s string) string { + // So, Afero's Glob likes to give "." as a name - that to list the "empty" dir name. + // GCS _really_ dislikes the dot and gives no entries for it - so we should rather replace the dot + // with an empty string + if s == "." { + return "" + } + return s +} + +func (fs *GcsFs) getObj(name string) stiface.ObjectHandle { + return fs.bucket.Object(name) +} + +func (fs *GcsFs) Name() string { return "GcsFs" } + +func (fs *GcsFs) Create(name string) (*GcsFile, error) { + name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))) + + if !fs.autoRemoveEmptyFolders { + baseDir := filepath.Base(name) + if stat, err := fs.Stat(baseDir); err != nil || !stat.IsDir() { + err = fs.MkdirAll(baseDir, 0) + if err != nil { + return nil, err + } + } + } + + obj := fs.getObj(name) + w := obj.NewWriter(fs.ctx) + var err error + err = w.Close() + if err != nil { + return nil, err + } + file := NewGcsFile(fs.ctx, fs, obj, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0, name) + + fs.rawGcsObjects[name] = file + return file, nil +} + +func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error { + name = fs.ensureTrailingSeparator(fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name)))) + + obj := fs.getObj(name) + w := obj.NewWriter(fs.ctx) + return w.Close() +} + +func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error { + path = fs.ensureTrailingSeparator(fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(path)))) + + root := "" + folders := strings.Split(path, fs.separator) + for i, f := range folders { + if f == "" && i != 0 { + continue // it's the last item - it should be empty + } + //Don't force a delimiter prefix + if root != "" { + root = root + fs.separator + f + } else { + root = f + } + + if err := fs.Mkdir(root, perm); err != nil { + return err + } + } + return nil +} + +func (fs *GcsFs) Open(name string) (*GcsFile, error) { + return fs.OpenFile(name, os.O_RDONLY, 0) +} + +func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) { + var file *GcsFile + var err error + + name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))) + + obj, found := fs.rawGcsObjects[name] + if found { + file = NewGcsFileFromOldFH(flag, fileMode, obj.resource) + } else { + file = NewGcsFile(fs.ctx, fs, fs.getObj(name), flag, fileMode, name) + } + + if flag == os.O_RDONLY { + _, err = file.Stat() + if err != nil { + return nil, err + } + } + + if flag&os.O_TRUNC != 0 { + err = file.resource.obj.Delete(fs.ctx) + if err != nil { + return nil, err + } + return fs.Create(name) + } + + if flag&os.O_APPEND != 0 { + _, err = file.Seek(0, 2) + if err != nil { + return nil, err + } + } + + if flag&os.O_CREATE != 0 { + _, err = file.Stat() + if err == nil { // the file actually exists + return nil, syscall.EPERM + } + + _, err = file.WriteString("") + if err != nil { + return nil, err + } + } + return file, nil +} + +func (fs *GcsFs) Remove(name string) error { + name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))) + + obj := fs.getObj(name) + info, err := fs.Stat(name) + if err != nil { + return err + } + delete(fs.rawGcsObjects, name) + + if info.IsDir() { + // it's a folder, we ha to check its contents - it cannot be removed, if not empty + var dir *GcsFile + dir, err = fs.Open(name) + if err != nil { + return err + } + var infos []os.FileInfo + infos, err = dir.Readdir(0) + if len(infos) > 0 { + return syscall.ENOTEMPTY + } + + // it's an empty folder, we can continue + name = fs.ensureTrailingSeparator(name) + obj = fs.getObj(name) + + return obj.Delete(fs.ctx) + } + return obj.Delete(fs.ctx) +} + +func (fs *GcsFs) RemoveAll(path string) error { + path = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(path))) + + pathInfo, err := fs.Stat(path) + if err != nil { + return err + } + if !pathInfo.IsDir() { + return fs.Remove(path) + } + + var dir *GcsFile + dir, err = fs.Open(path) + if err != nil { + return err + } + + var infos []os.FileInfo + infos, err = dir.Readdir(0) + for _, info := range infos { + err = fs.RemoveAll(path + fs.separator + info.Name()) + if err != nil { + return err + } + } + + return fs.Remove(path) + + //it := fs.bucket.Objects(fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: path, Versions: false}) + //for { + // objAttrs, err := it.Next() + // if err == iterator.Done { + // break + // } + // if err != nil { + // return err + // } + // + // name := objAttrs.Name + // if name == "" { + // name = objAttrs.Prefix + // } + // + // if name == path { + // // somehow happens + // continue + // } + // if objAttrs.Name == "" && objAttrs.Prefix != "" { + // // it's a folder, let's try to remove it normally first + // err = fs.Remove(path + fs.separator + objAttrs.Name) + // if err != nil { + // if err == syscall.ENOTEMPTY { + // err = fs.RemoveAll(path + fs.separator + objAttrs.Name) + // } + // } + // if err != nil { + // return err + // } + // + // } else { + // err = fs.Remove(objAttrs.Name) + // if err != nil { + // return err + // } + // } + //} + //return nil +} + +func (fs *GcsFs) Rename(oldName, newName string) error { + oldName = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(oldName))) + newName = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(newName))) + + src := fs.bucket.Object(oldName) + dst := fs.bucket.Object(newName) + + if _, err := dst.CopierFrom(src).Run(fs.ctx); err != nil { + return err + } + delete(fs.rawGcsObjects, oldName) + return src.Delete(fs.ctx) +} + +func (fs *GcsFs) Stat(name string) (os.FileInfo, error) { + name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))) + + return newFileInfo(name, fs, defaultFileMode) +} + +func (fs *GcsFs) Chmod(_ string, _ os.FileMode) error { + return errors.New("method Chmod is not implemented in GCS") +} + +func (fs *GcsFs) Chtimes(_ string, _, _ time.Time) error { + return errors.New("method Chtimes is not implemented. Create, Delete, Updated times are read only fields in GCS and set implicitly") +} + +func (fs *GcsFs) Chown(_ string, _, _ int) error { + return errors.New("method Chown is not implemented for GCS") +} diff --git a/go.mod b/go.mod index abe4fe1c..8c17b678 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,13 @@ module github.com/spf13/afero require ( + cloud.google.com/go/storage v1.14.0 + github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 github.com/pkg/sftp v1.10.1 - golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 - golang.org/x/text v0.3.3 + github.com/stretchr/testify v1.5.1 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/text v0.3.4 + google.golang.org/api v0.40.0 ) go 1.13 diff --git a/go.sum b/go.sum index 89d9bfbc..d7a83890 100644 --- a/go.sum +++ b/go.sum @@ -1,29 +1,459 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0 h1:XgtDnVJRCPEUG21gjFiRPz4zI1Mjg16R+NYQjfmU4XY= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 h1:tlyzajkF3030q6M8SvmJSemC9DTHL/xaMa18b65+JM4= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 h1:7KByu05hhLed2MO29w7p1XfZvZ13m8mub3shuVftRs0= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 h1:5vD4XjIc0X5+kHZjx4UecYdjA6mJo+XXNoaW0EjU5Os= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0 h1:uWrpz12dpVPn7cojP82mk02XDgTJLDPc2KbVTxrWb4A= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705 h1:PYBmACG+YEv8uQPW0r1kJj8tR+gkF0UWq7iFdUezwEw= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 78acf3b074293cfcf0b7b9a1a34cb2176a34b105 Mon Sep 17 00:00:00 2001 From: Vasily Ovchinnikov Date: Mon, 12 Apr 2021 20:01:12 +0200 Subject: [PATCH 2/6] Changed abstraction to operate on client, not bucket --- gcs.go | 33 ++- gcs_mocks.go | 8 + gcs_test.go | 637 ++++++++++++++++++++++++++++----------------- gcsfs/errors.go | 1 + gcsfs/file.go | 10 +- gcsfs/file_info.go | 12 +- gcsfs/fs.go | 210 ++++++++------- go.mod | 1 + 8 files changed, 562 insertions(+), 350 deletions(-) diff --git a/gcs.go b/gcs.go index a9787350..fd1f7175 100644 --- a/gcs.go +++ b/gcs.go @@ -21,9 +21,11 @@ import ( "os" "time" + "github.com/spf13/afero/gcsfs" + "cloud.google.com/go/storage" "github.com/googleapis/google-cloud-go-testing/storage/stiface" - "github.com/spf13/afero/gcsfs" + "google.golang.org/api/option" ) @@ -31,47 +33,44 @@ type GcsFs struct { source *gcsfs.GcsFs } -// Creates a GCS file system, automatically instantiating and decorating the storage client. +// NewGcsFS creates a GCS file system, automatically instantiating and decorating the storage client. // You can provide additional options to be passed to the client creation, as per // cloud.google.com/go/storage documentation -func NewGcsFS(ctx context.Context, bucketName string, opts ...option.ClientOption) (Fs, error) { +func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) { client, err := storage.NewClient(ctx, opts...) if err != nil { return nil, err } - return NewGcsFSFromClient(ctx, client, bucketName) + return NewGcsFSFromClient(ctx, client) } -// The same as NewGcsFS, but the files system will use the provided folder separator. -func NewGcsFSWithSeparator(ctx context.Context, bucketName, folderSeparator string, opts ...option.ClientOption) (Fs, error) { +// NewGcsFSWithSeparator is the same as NewGcsFS, but the files system will use the provided folder separator. +func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (Fs, error) { client, err := storage.NewClient(ctx, opts...) if err != nil { return nil, err } - return NewGcsFSFromClientWithSeparator(ctx, client, bucketName, folderSeparator) + return NewGcsFSFromClientWithSeparator(ctx, client, folderSeparator) } -// Creates a GCS file system from a given storage client -func NewGcsFSFromClient(ctx context.Context, client *storage.Client, bucketName string) (Fs, error) { +// NewGcsFSFromClient creates a GCS file system from a given storage client +func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (Fs, error) { c := stiface.AdaptClient(client) - bucket := c.Bucket(bucketName) - - return &GcsFs{gcsfs.NewGcsFs(ctx, bucket)}, nil + return &GcsFs{gcsfs.NewGcsFs(ctx, c)}, nil } -// Same as NewGcsFSFromClient, but the file system will use the provided folder separator. -func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, bucketName, folderSeparator string) (Fs, error) { +// NewGcsFSFromClientWithSeparator is the same as NewGcsFSFromClient, but the file system will use the provided folder separator. +func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (Fs, error) { c := stiface.AdaptClient(client) - bucket := c.Bucket(bucketName) - - return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, bucket, folderSeparator)}, nil + return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil } // Wraps gcs.GcsFs and convert some return types to afero interfaces. + func (fs *GcsFs) Name() string { return fs.source.Name() } diff --git a/gcs_mocks.go b/gcs_mocks.go index 2d6928e3..acbb1e5b 100644 --- a/gcs_mocks.go +++ b/gcs_mocks.go @@ -50,6 +50,10 @@ type bucketMock struct { fs Fs } +func (m *bucketMock) Attrs(context.Context) (*storage.BucketAttrs, error) { + return &storage.BucketAttrs{}, nil +} + func (m *bucketMock) Object(name string) stiface.ObjectHandle { return &objectMock{name: name, fs: m.fs} } @@ -193,6 +197,10 @@ func (r *readerMock) Read(p []byte) (int, error) { return r.file.Read(p) } +func (r *readerMock) Close() error { + return r.file.Close() +} + type objectItMock struct { stiface.ObjectIterator diff --git a/gcs_test.go b/gcs_test.go index 3b701e1f..58be450f 100644 --- a/gcs_test.go +++ b/gcs_test.go @@ -17,6 +17,8 @@ import ( "syscall" "testing" + "golang.org/x/oauth2/google" + "github.com/spf13/afero/gcsfs" "cloud.google.com/go/storage" @@ -28,6 +30,8 @@ const ( dirSize = 42 ) +var bucketName = "a-test-bucket" + var files = []struct { name string exists bool @@ -37,13 +41,14 @@ var files = []struct { offset int64 contentAtOffset string }{ - {"", true, true, dirSize, "", 0, ""}, // this is NOT a valid path for GCS, so we do some magic here {"sub", true, true, dirSize, "", 0, ""}, {"sub/testDir2", true, true, dirSize, "", 0, ""}, {"sub/testDir2/testFile", true, false, 8 * 1024, "c", 4 * 1024, "d"}, {"testFile", true, false, 12 * 1024, "a", 7 * 1024, "b"}, {"testDir1/testFile", true, false, 3 * 512, "b", 512, "c"}, + {"", false, true, dirSize, "", 0, ""}, // special case + {"nonExisting", false, false, dirSize, "", 0, ""}, } @@ -51,7 +56,7 @@ var dirs = []struct { name string children []string }{ - {"", []string{"sub", "testDir1", "testFile"}}, + {"", []string{"sub", "testDir1", "testFile"}}, // in this case it will be prepended with bucket name {"sub", []string{"testDir2"}}, {"sub/testDir2", []string{"testFile"}}, {"testDir1", []string{"testFile"}}, @@ -63,20 +68,35 @@ func TestMain(m *testing.M) { ctx := context.Background() var err error - // Check if GOOGLE_APPLICATION_CREDENTIALS are present. If not, then a fake service account + // in order to respect deferring + var exitCode int + defer os.Exit(exitCode) + + defer func() { + err := recover() + if err != nil { + fmt.Print(err) + exitCode = 2 + } + }() + + // Check if any credentials are present. If not, a fake service account, taken from the link // would be used: https://github.com/google/oauth2l/blob/master/integration/fixtures/fake-service-account.json - if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { + cred, err := google.FindDefaultCredentials(ctx) + if err != nil && !strings.HasPrefix(err.Error(), "google: could not find default credentials") { + panic(err) + } + + if cred == nil { var fakeCredentialsAbsPath string fakeCredentialsAbsPath, err = filepath.Abs("gcs-fake-service-account.json") if err != nil { - fmt.Print(err) - os.Exit(1) + panic(err) } err = os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", fakeCredentialsAbsPath) if err != nil { - fmt.Print(err) - os.Exit(1) + panic(err) } // reset it after the run @@ -92,8 +112,7 @@ func TestMain(m *testing.M) { var c *storage.Client c, err = storage.NewClient(ctx) if err != nil { - fmt.Print(err) - os.Exit(1) + panic(err) } client := stiface.AdaptClient(c) @@ -101,18 +120,12 @@ func TestMain(m *testing.M) { mockClient := newClientMock() mockClient.Client = client - bucket := mockClient.Bucket("a-test-bucket") + gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, mockClient)}} - // If you want to run the test suite on a LIVE bucket, comment the previous - // block and uncomment the line below and put your bucket name there. - // Keep in mind, that GCS will likely rate limit you, so it would be impossible - // to run the entire suite at once, only test by test. - //bucket := client.Bucket("a-test-bucket") + // Uncomment to use the real, not mocked, client + //gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, client)}} - gcsAfs = &Afero{Fs: &GcsFs{gcsfs.NewGcsFs(ctx, bucket)}} - - // defer here to assure our Env cleanup happens, if the mock was used - defer os.Exit(m.Run()) + exitCode = m.Run() } func createFiles(t *testing.T) { @@ -122,8 +135,10 @@ func createFiles(t *testing.T) { // the files have to be created first for _, f := range files { if !f.isdir && f.exists { + name := filepath.Join(bucketName, f.name) + var freshFile File - freshFile, err = gcsAfs.Create(f.name) + freshFile, err = gcsAfs.Create(name) if err != nil { t.Fatalf("failed to create a file \"%s\": %s", f.name, err) } @@ -160,7 +175,9 @@ func removeFiles(t *testing.T) { // the files have to be created first for _, f := range files { if !f.isdir && f.exists { - err = gcsAfs.Remove(f.name) + name := filepath.Join(bucketName, f.name) + + err = gcsAfs.Remove(name) if err != nil && err == syscall.ENOENT { t.Errorf("failed to remove file \"%s\": %s", f.name, err) } @@ -168,43 +185,55 @@ func removeFiles(t *testing.T) { } } -func TestFsOpen(t *testing.T) { +func TestGcsFsOpen(t *testing.T) { createFiles(t) defer removeFiles(t) for _, f := range files { - file, err := gcsAfs.Open(f.name) - if (err == nil) != f.exists { - t.Errorf("%v exists = %v, but got err = %v", f.name, f.exists, err) - } + nameBase := filepath.Join(bucketName, f.name) - if !f.exists { - continue + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - if err != nil { - t.Fatalf("%v: %v", f.name, err) + if f.name == "" { + names = []string{f.name} } - if file.Name() != filepath.FromSlash(f.name) { - t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(f.name)) - } + for _, name := range names { + file, err := gcsAfs.Open(name) + if (err == nil) != f.exists { + t.Errorf("%v exists = %v, but got err = %v", name, f.exists, err) + } - s, err := file.Stat() - if err != nil { - t.Fatalf("stat %v: got error '%v'", file.Name(), err) - } + if !f.exists { + continue + } + if err != nil { + t.Fatalf("%v: %v", name, err) + } - if isdir := s.IsDir(); isdir != f.isdir { - t.Errorf("%v directory, got: %v, expected: %v", file.Name(), isdir, f.isdir) - } + if file.Name() != filepath.FromSlash(nameBase) { + t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(nameBase)) + } + + s, err := file.Stat() + if err != nil { + t.Fatalf("stat %v: got error '%v'", file.Name(), err) + } + + if isdir := s.IsDir(); isdir != f.isdir { + t.Errorf("%v directory, got: %v, expected: %v", file.Name(), isdir, f.isdir) + } - if size := s.Size(); size != f.size { - t.Errorf("%v size, got: %v, expected: %v", file.Name(), size, f.size) + if size := s.Size(); size != f.size { + t.Errorf("%v size, got: %v, expected: %v", file.Name(), size, f.size) + } } } } -func TestRead(t *testing.T) { +func TestGcsRead(t *testing.T) { createFiles(t) defer removeFiles(t) @@ -213,25 +242,36 @@ func TestRead(t *testing.T) { continue } - file, err := gcsAfs.Open(f.name) - if err != nil { - t.Fatalf("opening %v: %v", f.name, err) + nameBase := filepath.Join(bucketName, f.name) + + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, + } + if f.name == "" { + names = []string{f.name} } - buf := make([]byte, 8) - n, err := file.Read(buf) - if err != nil { - if f.isdir && (err != syscall.EISDIR) { - t.Errorf("%v got error %v, expected EISDIR", f.name, err) - } else if !f.isdir { - t.Errorf("%v: %v", f.name, err) + for _, name := range names { + file, err := gcsAfs.Open(name) + if err != nil { + t.Fatalf("opening %v: %v", name, err) } - } else if n != 8 { - t.Errorf("%v: got %d read bytes, expected 8", f.name, n) - } else if string(buf) != strings.Repeat(f.content, testBytes) { - t.Errorf("%v: got <%s>, expected <%s>", f.name, f.content, string(buf)) - } + buf := make([]byte, 8) + n, err := file.Read(buf) + if err != nil { + if f.isdir && (err != syscall.EISDIR) { + t.Errorf("%v got error %v, expected EISDIR", name, err) + } else if !f.isdir { + t.Errorf("%v: %v", name, err) + } + } else if n != 8 { + t.Errorf("%v: got %d read bytes, expected 8", name, n) + } else if string(buf) != strings.Repeat(f.content, testBytes) { + t.Errorf("%v: got <%s>, expected <%s>", f.name, f.content, string(buf)) + } + } } } @@ -244,25 +284,36 @@ func TestGcsReadAt(t *testing.T) { continue } - file, err := gcsAfs.Open(f.name) - if err != nil { - t.Fatalf("opening %v: %v", f.name, err) + nameBase := filepath.Join(bucketName, f.name) + + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, + } + if f.name == "" { + names = []string{f.name} } - buf := make([]byte, testBytes) - n, err := file.ReadAt(buf, f.offset-testBytes/2) - if err != nil { - if f.isdir && (err != syscall.EISDIR) { - t.Errorf("%v got error %v, expected EISDIR", f.name, err) - } else if !f.isdir { - t.Errorf("%v: %v", f.name, err) + for _, name := range names { + file, err := gcsAfs.Open(name) + if err != nil { + t.Fatalf("opening %v: %v", name, err) } - } else if n != 8 { - t.Errorf("%v: got %d read bytes, expected 8", f.name, n) - } else if string(buf) != strings.Repeat(f.content, testBytes/2)+strings.Repeat(f.contentAtOffset, testBytes/2) { - t.Errorf("%v: got <%s>, expected <%s>", f.name, f.contentAtOffset, string(buf)) - } + buf := make([]byte, testBytes) + n, err := file.ReadAt(buf, f.offset-testBytes/2) + if err != nil { + if f.isdir && (err != syscall.EISDIR) { + t.Errorf("%v got error %v, expected EISDIR", name, err) + } else if !f.isdir { + t.Errorf("%v: %v", name, err) + } + } else if n != 8 { + t.Errorf("%v: got %d read bytes, expected 8", f.name, n) + } else if string(buf) != strings.Repeat(f.content, testBytes/2)+strings.Repeat(f.contentAtOffset, testBytes/2) { + t.Errorf("%v: got <%s>, expected <%s>", f.name, f.contentAtOffset, string(buf)) + } + } } } @@ -275,43 +326,55 @@ func TestGcsSeek(t *testing.T) { continue } - file, err := gcsAfs.Open(f.name) - if err != nil { - t.Fatalf("opening %v: %v", f.name, err) - } + nameBase := filepath.Join(bucketName, f.name) - var tests = []struct { - offIn int64 - whence int - offOut int64 - }{ - {0, io.SeekStart, 0}, - {10, io.SeekStart, 10}, - {1, io.SeekCurrent, 11}, - {10, io.SeekCurrent, 21}, - {0, io.SeekEnd, f.size}, - {-1, io.SeekEnd, f.size - 1}, + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, + } + if f.name == "" { + names = []string{f.name} } - for _, s := range tests { - n, err := file.Seek(s.offIn, s.whence) + for _, name := range names { + file, err := gcsAfs.Open(name) if err != nil { - if f.isdir && err == syscall.EISDIR { - continue - } + t.Fatalf("opening %v: %v", name, err) + } - t.Errorf("%v: %v", f.name, err) + var tests = []struct { + offIn int64 + whence int + offOut int64 + }{ + {0, io.SeekStart, 0}, + {10, io.SeekStart, 10}, + {1, io.SeekCurrent, 11}, + {10, io.SeekCurrent, 21}, + {0, io.SeekEnd, f.size}, + {-1, io.SeekEnd, f.size - 1}, } - if n != s.offOut { - t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offIn, s.whence, n, s.offOut) + for _, s := range tests { + n, err := file.Seek(s.offIn, s.whence) + if err != nil { + if f.isdir && err == syscall.EISDIR { + continue + } + + t.Errorf("%v: %v", name, err) + } + + if n != s.offOut { + t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offIn, s.whence, n, s.offOut) + } } } } } -func TestName(t *testing.T) { +func TestGcsName(t *testing.T) { createFiles(t) defer removeFiles(t) @@ -320,20 +383,32 @@ func TestName(t *testing.T) { continue } - file, err := gcsAfs.Open(f.name) - if err != nil { - t.Fatalf("opening %v: %v", f.name, err) + nameBase := filepath.Join(bucketName, f.name) + + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } + if f.name == "" { + names = []string{f.name} + } + + for _, name := range names { + file, err := gcsAfs.Open(name) + if err != nil { + t.Fatalf("opening %v: %v", name, err) + } - n := file.Name() - if n != filepath.FromSlash(f.name) { - t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(f.name)) + n := file.Name() + if n != filepath.FromSlash(nameBase) { + t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(nameBase)) + } } } } -func TestClose(t *testing.T) { +func TestGcsClose(t *testing.T) { createFiles(t) defer removeFiles(t) @@ -342,35 +417,47 @@ func TestClose(t *testing.T) { continue } - file, err := gcsAfs.Open(f.name) - if err != nil { - t.Fatalf("opening %v: %v", f.name, err) - } + nameBase := filepath.Join(bucketName, f.name) - err = file.Close() - if err != nil { - t.Errorf("%v: %v", f.name, err) + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - - err = file.Close() - if err == nil { - t.Errorf("%v: closing twice should return an error", f.name) + if f.name == "" { + names = []string{f.name} } - buf := make([]byte, 8) - n, err := file.Read(buf) - if n != 0 || err == nil { - t.Errorf("%v: could read from a closed file", f.name) - } + for _, name := range names { + file, err := gcsAfs.Open(name) + if err != nil { + t.Fatalf("opening %v: %v", name, err) + } - n, err = file.ReadAt(buf, 256) - if n != 0 || err == nil { - t.Errorf("%v: could readAt from a closed file", f.name) - } + err = file.Close() + if err != nil { + t.Errorf("%v: %v", name, err) + } - off, err := file.Seek(0, io.SeekStart) - if off != 0 || err == nil { - t.Errorf("%v: could seek from a closed file", f.name) + err = file.Close() + if err == nil { + t.Errorf("%v: closing twice should return an error", name) + } + + buf := make([]byte, 8) + n, err := file.Read(buf) + if n != 0 || err == nil { + t.Errorf("%v: could read from a closed file", name) + } + + n, err = file.ReadAt(buf, 256) + if n != 0 || err == nil { + t.Errorf("%v: could readAt from a closed file", name) + } + + off, err := file.Seek(0, io.SeekStart) + if off != 0 || err == nil { + t.Errorf("%v: could seek from a closed file", name) + } } } } @@ -380,56 +467,81 @@ func TestGcsOpenFile(t *testing.T) { defer removeFiles(t) for _, f := range files { - file, err := gcsAfs.OpenFile(f.name, os.O_RDONLY, 0400) - if !f.exists { - if !errors.Is(err, syscall.ENOENT) { - t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT) - } + nameBase := filepath.Join(bucketName, f.name) - continue + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - - if err != nil { - t.Fatalf("%v: %v", f.name, err) + if f.name == "" { + names = []string{f.name} } - err = file.Close() - if err != nil { - t.Fatalf("failed to close a file \"%s\": %s", f.name, err) - } + for _, name := range names { + file, err := gcsAfs.OpenFile(name, os.O_RDONLY, 0400) + if !f.exists { + if (f.name != "" && !errors.Is(err, syscall.ENOENT)) || + (f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) { + t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT) + } - file, err = gcsAfs.OpenFile(f.name, os.O_CREATE, 0600) - if !errors.Is(err, syscall.EPERM) { - t.Errorf("%v: open for write: got %v, expected %v", f.name, err, syscall.EPERM) - } + continue + } + + if err != nil { + t.Fatalf("%v: %v", name, err) + } + err = file.Close() + if err != nil { + t.Fatalf("failed to close a file \"%s\": %s", name, err) + } + + file, err = gcsAfs.OpenFile(name, os.O_CREATE, 0600) + if !errors.Is(err, syscall.EPERM) { + t.Errorf("%v: open for write: got %v, expected %v", name, err, syscall.EPERM) + } + } } } -func TestFsStat(t *testing.T) { +func TestGcsFsStat(t *testing.T) { createFiles(t) defer removeFiles(t) for _, f := range files { - fi, err := gcsAfs.Stat(f.name) - if !f.exists { - if !errors.Is(err, syscall.ENOENT) { - t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT) - } + nameBase := filepath.Join(bucketName, f.name) - continue + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - - if err != nil { - t.Fatalf("stat %v: got error '%v'", f.name, err) + if f.name == "" { + names = []string{f.name} } - if isdir := fi.IsDir(); isdir != f.isdir { - t.Errorf("%v directory, got: %v, expected: %v", f.name, isdir, f.isdir) - } + for _, name := range names { + fi, err := gcsAfs.Stat(name) + if !f.exists { + if (f.name != "" && !errors.Is(err, syscall.ENOENT)) || + (f.name == "" && !errors.Is(err, gcsfs.ErrNoBucketInName)) { + t.Errorf("%v: got %v, expected%v", name, err, syscall.ENOENT) + } + + continue + } + + if err != nil { + t.Fatalf("stat %v: got error '%v'", name, err) + } + + if isdir := fi.IsDir(); isdir != f.isdir { + t.Errorf("%v directory, got: %v, expected: %v", name, isdir, f.isdir) + } - if size := fi.Size(); size != f.size { - t.Errorf("%v size, got: %v, expected: %v", f.name, size, f.size) + if size := fi.Size(); size != f.size { + t.Errorf("%v size, got: %v, expected: %v", name, size, f.size) + } } } } @@ -439,47 +551,65 @@ func TestGcsReaddir(t *testing.T) { defer removeFiles(t) for _, d := range dirs { - dir, err := gcsAfs.Open(d.name) - if err != nil { - t.Fatal(err) - } + nameBase := filepath.Join(bucketName, d.name) - fi, err := dir.Readdir(0) - if err != nil { - t.Fatal(err) - } - var names []string - for _, f := range fi { - names = append(names, f.Name()) + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - if !reflect.DeepEqual(names, d.children) { - t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children) - } + for _, name := range names { + dir, err := gcsAfs.Open(name) + if err != nil { + t.Fatal(err) + } - fi, err = dir.Readdir(1) - if err != nil { - t.Fatal(err) - } + fi, err := dir.Readdir(0) + if err != nil { + t.Fatal(err) + } + var fileNames []string + for _, f := range fi { + fileNames = append(fileNames, f.Name()) + } - names = []string{} - for _, f := range fi { - names = append(names, f.Name()) - } + if !reflect.DeepEqual(fileNames, d.children) { + t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children) + } + + fi, err = dir.Readdir(1) + if err != nil { + t.Fatal(err) + } + + fileNames = []string{} + for _, f := range fi { + fileNames = append(fileNames, f.Name()) + } - if !reflect.DeepEqual(names, d.children[0:1]) { - t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1]) + if !reflect.DeepEqual(fileNames, d.children[0:1]) { + t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children[0:1]) + } } } - dir, err := gcsAfs.Open("testFile") - if err != nil { - t.Fatal(err) + nameBase := filepath.Join(bucketName, "testFile") + + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - _, err = dir.Readdir(-1) - if err != syscall.ENOTDIR { - t.Fatal("Expected error") + for _, name := range names { + dir, err := gcsAfs.Open(name) + if err != nil { + t.Fatal(err) + } + + _, err = dir.Readdir(-1) + if err != syscall.ENOTDIR { + t.Fatal("Expected error") + } } } @@ -488,38 +618,56 @@ func TestGcsReaddirnames(t *testing.T) { defer removeFiles(t) for _, d := range dirs { - dir, err := gcsAfs.Open(d.name) - if err != nil { - t.Fatal(err) - } + nameBase := filepath.Join(bucketName, d.name) - names, err := dir.Readdirnames(0) - if err != nil { - t.Fatal(err) + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - if !reflect.DeepEqual(names, d.children) { - t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children) - } + for _, name := range names { + dir, err := gcsAfs.Open(name) + if err != nil { + t.Fatal(err) + } - names, err = dir.Readdirnames(1) - if err != nil { - t.Fatal(err) - } + fileNames, err := dir.Readdirnames(0) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(fileNames, d.children) { + t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children) + } - if !reflect.DeepEqual(names, d.children[0:1]) { - t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1]) + fileNames, err = dir.Readdirnames(1) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(fileNames, d.children[0:1]) { + t.Errorf("%v: children, got '%v', expected '%v'", name, fileNames, d.children[0:1]) + } } } - dir, err := gcsAfs.Open("testFile") - if err != nil { - t.Fatal(err) + nameBase := filepath.Join(bucketName, "testFile") + + names := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - _, err = dir.Readdir(-1) - if err != syscall.ENOTDIR { - t.Fatal("Expected error") + for _, name := range names { + dir, err := gcsAfs.Open(name) + if err != nil { + t.Fatal(err) + } + + _, err = dir.Readdirnames(-1) + if err != syscall.ENOTDIR { + t.Fatal("Expected error") + } } } @@ -531,29 +679,40 @@ func TestGcsGlob(t *testing.T) { glob string entries []string }{ - {filepath.FromSlash("/*"), []string{filepath.FromSlash("/sub"), filepath.FromSlash("/testDir1"), filepath.FromSlash("/testFile")}}, {filepath.FromSlash("*"), []string{filepath.FromSlash("sub"), filepath.FromSlash("testDir1"), filepath.FromSlash("testFile")}}, - {filepath.FromSlash("/sub/*"), []string{filepath.FromSlash("/sub/testDir2")}}, - {filepath.FromSlash("/sub/testDir2/*"), []string{filepath.FromSlash("/sub/testDir2/testFile")}}, - {filepath.FromSlash("/testDir1/*"), []string{filepath.FromSlash("/testDir1/testFile")}}, {filepath.FromSlash("sub/*"), []string{filepath.FromSlash("sub/testDir2")}}, {filepath.FromSlash("sub/testDir2/*"), []string{filepath.FromSlash("sub/testDir2/testFile")}}, {filepath.FromSlash("testDir1/*"), []string{filepath.FromSlash("testDir1/testFile")}}, } { - entries, err := Glob(gcsAfs.Fs, s.glob) - if err != nil { - t.Error(err) + nameBase := filepath.Join(bucketName, s.glob) + + prefixedGlobs := []string{ + nameBase, + string(os.PathSeparator) + nameBase, } - if reflect.DeepEqual(entries, s.entries) { - t.Logf("glob: %s: glob ok", s.glob) - } else { - t.Errorf("glob: %s: got %#v, expected %#v", s.glob, entries, s.entries) + + prefixedEntries := [][]string{{}, {}} + for _, entry := range s.entries { + prefixedEntries[0] = append(prefixedEntries[0], filepath.Join(bucketName, entry)) + prefixedEntries[1] = append(prefixedEntries[1], string(os.PathSeparator)+filepath.Join(bucketName, entry)) + } + + for i, prefixedGlob := range prefixedGlobs { + entries, err := Glob(gcsAfs.Fs, prefixedGlob) + if err != nil { + t.Error(err) + } + if reflect.DeepEqual(entries, prefixedEntries[i]) { + t.Logf("glob: %s: glob ok", prefixedGlob) + } else { + t.Errorf("glob: %s: got %#v, expected %#v", prefixedGlob, entries, prefixedEntries) + } } } } -func TestMkdir(t *testing.T) { - dirName := "/a-test-dir" +func TestGcsMkdir(t *testing.T) { + dirName := filepath.Join(bucketName, "a-test-dir") var err error err = gcsAfs.Mkdir(dirName, 0755) @@ -582,45 +741,47 @@ func TestMkdir(t *testing.T) { } } -func TestMkdirAll(t *testing.T) { - err := gcsAfs.MkdirAll("/a/b/c", 0755) +func TestGcsMkdirAll(t *testing.T) { + dirName := filepath.Join(bucketName, "a/b/c") + + err := gcsAfs.MkdirAll(dirName, 0755) if err != nil { t.Fatal(err) } - info, err := gcsAfs.Stat("/a") + info, err := gcsAfs.Stat(filepath.Join(bucketName, "a")) if err != nil { t.Fatal(err) } if !info.Mode().IsDir() { - t.Error("/a: mode is not directory") + t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a")) } if info.Mode() != os.ModeDir|0755 { - t.Errorf("/a: wrong permissions, expected drwxr-xr-x, got %s", info.Mode()) + t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a"), info.Mode()) } - info, err = gcsAfs.Stat("/a/b") + info, err = gcsAfs.Stat(filepath.Join(bucketName, "a/b")) if err != nil { t.Fatal(err) } if !info.Mode().IsDir() { - t.Error("/a/b: mode is not directory") + t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a/b")) } if info.Mode() != os.ModeDir|0755 { - t.Errorf("/a/b: wrong permissions, expected drwxr-xr-x, got %s", info.Mode()) + t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a/b"), info.Mode()) } - info, err = gcsAfs.Stat("/a/b/c") + info, err = gcsAfs.Stat(dirName) if err != nil { t.Fatal(err) } if !info.Mode().IsDir() { - t.Error("/a/b/c: mode is not directory") + t.Errorf("%s: mode is not directory", dirName) } if info.Mode() != os.ModeDir|0755 { - t.Errorf("/a/b/c: wrong permissions, expected drwxr-xr-x, got %s", info.Mode()) + t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode()) } - err = gcsAfs.RemoveAll("/a") + err = gcsAfs.RemoveAll(filepath.Join(bucketName, "a")) if err != nil { - t.Fatalf("failed to remove the folder /a with error: %s", err) + t.Fatalf("failed to remove the folder %s with error: %s", filepath.Join(bucketName, "a"), err) } } diff --git a/gcsfs/errors.go b/gcsfs/errors.go index 7468ad5b..201cd676 100644 --- a/gcsfs/errors.go +++ b/gcsfs/errors.go @@ -22,6 +22,7 @@ import ( ) var ( + ErrNoBucketInName = errors.New("no bucket name found in the name") ErrFileClosed = errors.New("file is closed") ErrOutOfRange = errors.New("out of range") ErrObjectDoesNotExist = errors.New("storage: object doesn't exist") diff --git a/gcsfs/file.go b/gcsfs/file.go index 33364a4a..b3b3d892 100644 --- a/gcsfs/file.go +++ b/gcsfs/file.go @@ -195,11 +195,13 @@ func (o *GcsFile) readdirImpl(count int) ([]*FileInfo, error) { return nil, syscall.ENOTDIR } - path := o.resource.fs.ensureTrailingSeparator(o.Name()) + path := o.resource.fs.ensureTrailingSeparator(o.resource.name) if o.ReadDirIt == nil { //log.Printf("Querying path : %s\n", path) - o.ReadDirIt = o.resource.fs.bucket.Objects( - o.resource.ctx, &storage.Query{Delimiter: o.resource.fs.separator, Prefix: path, Versions: false}) + bucketName, bucketPath := o.resource.fs.splitName(path) + + o.ReadDirIt = o.resource.fs.client.Bucket(bucketName).Objects( + o.resource.ctx, &storage.Query{Delimiter: o.resource.fs.separator, Prefix: bucketPath, Versions: false}) } var res []*FileInfo for { @@ -283,7 +285,7 @@ func (o *GcsFile) Stat() (os.FileInfo, error) { return nil, err } - return newFileInfo(o.Name(), o.resource.fs, o.resource.fileMode) + return newFileInfo(o.resource.name, o.resource.fs, o.resource.fileMode) } func (o *GcsFile) Sync() error { diff --git a/gcsfs/file_info.go b/gcsfs/file_info.go index b14a5f6a..2cccb7ca 100644 --- a/gcsfs/file_info.go +++ b/gcsfs/file_info.go @@ -46,7 +46,10 @@ func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error fileMode: fileMode, } - obj := fs.getObj(name) + obj, err := fs.getObj(name) + if err != nil { + return nil, err + } objAttrs, err := obj.Attrs(fs.ctx) if err != nil { @@ -58,8 +61,9 @@ func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error } else if err.Error() == ErrObjectDoesNotExist.Error() { // Folders do not actually "exist" in GCloud, so we have to check, if something exists with // such a prefix - it := fs.bucket.Objects( - fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: name, Versions: false}) + bucketName, bucketPath := fs.splitName(name) + it := fs.client.Bucket(bucketName).Objects( + fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: bucketPath, Versions: false}) if _, err = it.Next(); err == nil { res.name = fs.ensureTrailingSeparator(res.name) res.isDir = true @@ -100,7 +104,7 @@ func newFileInfoFromAttrs(objAttrs *storage.ObjectAttrs, fileMode os.FileMode) * } func (fi *FileInfo) Name() string { - return filepath.Base(fi.name) + return filepath.Base(filepath.FromSlash(fi.name)) } func (fi *FileInfo) Size() int64 { diff --git a/gcsfs/fs.go b/gcsfs/fs.go index 79bfa3b7..ca452184 100644 --- a/gcsfs/fs.go +++ b/gcsfs/fs.go @@ -19,7 +19,6 @@ package gcsfs import ( "context" "errors" - "log" "os" "path/filepath" "strings" @@ -31,26 +30,29 @@ import ( const ( defaultFileMode = 0755 + gsPrefix = "gs://" ) // GcsFs is a Fs implementation that uses functions provided by google cloud storage type GcsFs struct { - ctx context.Context - bucket stiface.BucketHandle - separator string + ctx context.Context + client stiface.Client + separator string + + buckets map[string]stiface.BucketHandle rawGcsObjects map[string]*GcsFile autoRemoveEmptyFolders bool //trigger for creating "virtual folders" (not required by GCSs) } -func NewGcsFs(ctx context.Context, bucket stiface.BucketHandle) *GcsFs { - return NewGcsFsWithSeparator(ctx, bucket, "/") +func NewGcsFs(ctx context.Context, client stiface.Client) *GcsFs { + return NewGcsFsWithSeparator(ctx, client, "/") } -func NewGcsFsWithSeparator(ctx context.Context, bucket stiface.BucketHandle, folderSep string) *GcsFs { +func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *GcsFs { return &GcsFs{ ctx: ctx, - bucket: bucket, + client: client, separator: folderSep, rawGcsObjects: make(map[string]*GcsFile), @@ -69,39 +71,65 @@ func (fs *GcsFs) ensureTrailingSeparator(s string) string { } return s } - -func (fs *GcsFs) ensureNoLeadingSeparators(s string) string { - // GCS does REALLY not like the names, that begin with a separator +func (fs *GcsFs) ensureNoLeadingSeparator(s string) string { if len(s) > 0 && strings.HasPrefix(s, fs.separator) { - log.Printf( - "WARNING: the provided path \"%s\" starts with a separator \"%s\", which is not supported by "+ - "GCloud. The separator will be automatically trimmed", - s, - fs.separator, - ) - return s[len(fs.separator):] + s = s[len(fs.separator):] } + return s } -func correctTheDot(s string) string { - // So, Afero's Glob likes to give "." as a name - that to list the "empty" dir name. - // GCS _really_ dislikes the dot and gives no entries for it - so we should rather replace the dot - // with an empty string - if s == "." { - return "" +func ensureNoPrefix(s string) string { + if len(s) > 0 && strings.HasPrefix(s, gsPrefix) { + return s[len(gsPrefix):] } return s } -func (fs *GcsFs) getObj(name string) stiface.ObjectHandle { - return fs.bucket.Object(name) +func validateName(s string) error { + if len(s) == 0 { + return ErrNoBucketInName + } + return nil +} + +// Splits provided name into bucket name and path +func (fs *GcsFs) splitName(name string) (bucketName string, path string) { + splitName := strings.Split(name, fs.separator) + + return splitName[0], strings.Join(splitName[1:], fs.separator) +} + +func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) { + bucket := fs.buckets[name] + if bucket == nil { + bucket = fs.client.Bucket(name) + _, err := bucket.Attrs(fs.ctx) + if err != nil { + return nil, err + } + } + return bucket, nil +} + +func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) { + bucketName, path := fs.splitName(name) + + bucket, err := fs.getBucket(bucketName) + if err != nil { + return nil, err + } + + return bucket.Object(path), nil } func (fs *GcsFs) Name() string { return "GcsFs" } func (fs *GcsFs) Create(name string) (*GcsFile, error) { - name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))) + name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) + if err := validateName(name); err != nil { + return nil, err + } if !fs.autoRemoveEmptyFolders { baseDir := filepath.Base(name) @@ -113,9 +141,11 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) { } } - obj := fs.getObj(name) + obj, err := fs.getObj(name) + if err != nil { + return nil, err + } w := obj.NewWriter(fs.ctx) - var err error err = w.Close() if err != nil { return nil, err @@ -127,15 +157,24 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) { } func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error { - name = fs.ensureTrailingSeparator(fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name)))) + name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name)))) + if err := validateName(name); err != nil { + return err + } - obj := fs.getObj(name) + obj, err := fs.getObj(name) + if err != nil { + return err + } w := obj.NewWriter(fs.ctx) return w.Close() } func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error { - path = fs.ensureTrailingSeparator(fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(path)))) + path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path)))) + if err := validateName(path); err != nil { + return err + } root := "" folders := strings.Split(path, fs.separator) @@ -165,13 +204,21 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile var file *GcsFile var err error - name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))) + name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) + if err = validateName(name); err != nil { + return nil, err + } - obj, found := fs.rawGcsObjects[name] + f, found := fs.rawGcsObjects[name] if found { - file = NewGcsFileFromOldFH(flag, fileMode, obj.resource) + file = NewGcsFileFromOldFH(flag, fileMode, f.resource) } else { - file = NewGcsFile(fs.ctx, fs, fs.getObj(name), flag, fileMode, name) + var obj stiface.ObjectHandle + obj, err = fs.getObj(name) + if err != nil { + return nil, err + } + file = NewGcsFile(fs.ctx, fs, obj, flag, fileMode, name) } if flag == os.O_RDONLY { @@ -211,9 +258,15 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile } func (fs *GcsFs) Remove(name string) error { - name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))) + name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) + if err := validateName(name); err != nil { + return err + } - obj := fs.getObj(name) + obj, err := fs.getObj(name) + if err != nil { + return err + } info, err := fs.Stat(name) if err != nil { return err @@ -235,7 +288,10 @@ func (fs *GcsFs) Remove(name string) error { // it's an empty folder, we can continue name = fs.ensureTrailingSeparator(name) - obj = fs.getObj(name) + obj, err = fs.getObj(name) + if err != nil { + return err + } return obj.Delete(fs.ctx) } @@ -243,7 +299,10 @@ func (fs *GcsFs) Remove(name string) error { } func (fs *GcsFs) RemoveAll(path string) error { - path = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(path))) + path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path))) + if err := validateName(path); err != nil { + return err + } pathInfo, err := fs.Stat(path) if err != nil { @@ -262,63 +321,37 @@ func (fs *GcsFs) RemoveAll(path string) error { var infos []os.FileInfo infos, err = dir.Readdir(0) for _, info := range infos { - err = fs.RemoveAll(path + fs.separator + info.Name()) + nameToRemove := fs.normSeparators(info.Name()) + err = fs.RemoveAll(path + fs.separator + nameToRemove) if err != nil { return err } } return fs.Remove(path) - - //it := fs.bucket.Objects(fs.ctx, &storage.Query{Delimiter: fs.separator, Prefix: path, Versions: false}) - //for { - // objAttrs, err := it.Next() - // if err == iterator.Done { - // break - // } - // if err != nil { - // return err - // } - // - // name := objAttrs.Name - // if name == "" { - // name = objAttrs.Prefix - // } - // - // if name == path { - // // somehow happens - // continue - // } - // if objAttrs.Name == "" && objAttrs.Prefix != "" { - // // it's a folder, let's try to remove it normally first - // err = fs.Remove(path + fs.separator + objAttrs.Name) - // if err != nil { - // if err == syscall.ENOTEMPTY { - // err = fs.RemoveAll(path + fs.separator + objAttrs.Name) - // } - // } - // if err != nil { - // return err - // } - // - // } else { - // err = fs.Remove(objAttrs.Name) - // if err != nil { - // return err - // } - // } - //} - //return nil } func (fs *GcsFs) Rename(oldName, newName string) error { - oldName = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(oldName))) - newName = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(newName))) + oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName))) + if err := validateName(oldName); err != nil { + return err + } - src := fs.bucket.Object(oldName) - dst := fs.bucket.Object(newName) + newName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(newName))) + if err := validateName(newName); err != nil { + return err + } - if _, err := dst.CopierFrom(src).Run(fs.ctx); err != nil { + src, err := fs.getObj(oldName) + if err != nil { + return err + } + dst, err := fs.getObj(newName) + if err != nil { + return err + } + + if _, err = dst.CopierFrom(src).Run(fs.ctx); err != nil { return err } delete(fs.rawGcsObjects, oldName) @@ -326,7 +359,10 @@ func (fs *GcsFs) Rename(oldName, newName string) error { } func (fs *GcsFs) Stat(name string) (os.FileInfo, error) { - name = fs.ensureNoLeadingSeparators(fs.normSeparators(correctTheDot(name))) + name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name))) + if err := validateName(name); err != nil { + return nil, err + } return newFileInfo(name, fs, defaultFileMode) } diff --git a/go.mod b/go.mod index 8c17b678..e6cf780b 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/pkg/sftp v1.10.1 github.com/stretchr/testify v1.5.1 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99 // indirect golang.org/x/text v0.3.4 google.golang.org/api v0.40.0 ) From 13ee679c8e71e96f2938b758fc81a09fe59fc569 Mon Sep 17 00:00:00 2001 From: VOvchinnikov Date: Thu, 5 Aug 2021 18:24:24 +0200 Subject: [PATCH 3/6] Added test cases for `Mkdir` & `MkdirAll` to cover empty folder names --- gcs_mocks.go | 7 +++ gcs_test.go | 144 +++++++++++++++++++++++++++++---------------------- 2 files changed, 89 insertions(+), 62 deletions(-) diff --git a/gcs_mocks.go b/gcs_mocks.go index acbb1e5b..57e0da56 100644 --- a/gcs_mocks.go +++ b/gcs_mocks.go @@ -146,6 +146,10 @@ type writerMock struct { } func (w *writerMock) Write(p []byte) (n int, err error) { + if w.name == "" { + return 0, gcsfs.ErrEmptyObjectName + } + if w.file == nil { w.file, err = w.fs.Create(w.name) if err != nil { @@ -157,6 +161,9 @@ func (w *writerMock) Write(p []byte) (n int, err error) { } func (w *writerMock) Close() error { + if w.name == "" { + return gcsfs.ErrEmptyObjectName + } if w.file == nil { var err error if strings.HasSuffix(w.name, "/") { diff --git a/gcs_test.go b/gcs_test.go index 58be450f..37c2cf56 100644 --- a/gcs_test.go +++ b/gcs_test.go @@ -712,76 +712,96 @@ func TestGcsGlob(t *testing.T) { } func TestGcsMkdir(t *testing.T) { - dirName := filepath.Join(bucketName, "a-test-dir") - var err error + t.Run("empty", func(t *testing.T) { + emptyDirName := bucketName - err = gcsAfs.Mkdir(dirName, 0755) - if err != nil { - t.Fatal("failed to create a folder with error", err) - } + err := gcsAfs.Mkdir(emptyDirName, 0755) + if err == nil { + t.Fatal("did not fail upon creation of an empty folder") + } + }) + t.Run("success", func(t *testing.T) { + dirName := filepath.Join(bucketName, "a-test-dir") + var err error - info, err := gcsAfs.Stat(dirName) - if err != nil { - t.Fatal("failed to get info", err) - } - if !info.IsDir() { - t.Fatalf("%s: not a dir", dirName) - } - if !info.Mode().IsDir() { - t.Errorf("%s: mode is not directory", dirName) - } + err = gcsAfs.Mkdir(dirName, 0755) + if err != nil { + t.Fatal("failed to create a folder with error", err) + } - if info.Mode() != os.ModeDir|0755 { - t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode()) - } + info, err := gcsAfs.Stat(dirName) + if err != nil { + t.Fatal("failed to get info", err) + } + if !info.IsDir() { + t.Fatalf("%s: not a dir", dirName) + } + if !info.Mode().IsDir() { + t.Errorf("%s: mode is not directory", dirName) + } - err = gcsAfs.Remove(dirName) - if err != nil { - t.Fatalf("could not delete the folder %s after the test with error: %s", dirName, err) - } + if info.Mode() != os.ModeDir|0755 { + t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode()) + } + + err = gcsAfs.Remove(dirName) + if err != nil { + t.Fatalf("could not delete the folder %s after the test with error: %s", dirName, err) + } + }) } func TestGcsMkdirAll(t *testing.T) { - dirName := filepath.Join(bucketName, "a/b/c") + t.Run("empty", func(t *testing.T) { + emptyDirName := bucketName - err := gcsAfs.MkdirAll(dirName, 0755) - if err != nil { - t.Fatal(err) - } + err := gcsAfs.MkdirAll(emptyDirName, 0755) + if err == nil { + t.Fatal("did not fail upon creation of an empty folder") + } + }) + t.Run("success", func(t *testing.T) { + dirName := filepath.Join(bucketName, "a/b/c") - info, err := gcsAfs.Stat(filepath.Join(bucketName, "a")) - if err != nil { - t.Fatal(err) - } - if !info.Mode().IsDir() { - t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a")) - } - if info.Mode() != os.ModeDir|0755 { - t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a"), info.Mode()) - } - info, err = gcsAfs.Stat(filepath.Join(bucketName, "a/b")) - if err != nil { - t.Fatal(err) - } - if !info.Mode().IsDir() { - t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a/b")) - } - if info.Mode() != os.ModeDir|0755 { - t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a/b"), info.Mode()) - } - info, err = gcsAfs.Stat(dirName) - if err != nil { - t.Fatal(err) - } - if !info.Mode().IsDir() { - t.Errorf("%s: mode is not directory", dirName) - } - if info.Mode() != os.ModeDir|0755 { - t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode()) - } + err := gcsAfs.MkdirAll(dirName, 0755) + if err != nil { + t.Fatal(err) + } - err = gcsAfs.RemoveAll(filepath.Join(bucketName, "a")) - if err != nil { - t.Fatalf("failed to remove the folder %s with error: %s", filepath.Join(bucketName, "a"), err) - } + info, err := gcsAfs.Stat(filepath.Join(bucketName, "a")) + if err != nil { + t.Fatal(err) + } + if !info.Mode().IsDir() { + t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a")) + } + if info.Mode() != os.ModeDir|0755 { + t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a"), info.Mode()) + } + info, err = gcsAfs.Stat(filepath.Join(bucketName, "a/b")) + if err != nil { + t.Fatal(err) + } + if !info.Mode().IsDir() { + t.Errorf("%s: mode is not directory", filepath.Join(bucketName, "a/b")) + } + if info.Mode() != os.ModeDir|0755 { + t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", filepath.Join(bucketName, "a/b"), info.Mode()) + } + info, err = gcsAfs.Stat(dirName) + if err != nil { + t.Fatal(err) + } + if !info.Mode().IsDir() { + t.Errorf("%s: mode is not directory", dirName) + } + if info.Mode() != os.ModeDir|0755 { + t.Errorf("%s: wrong permissions, expected drwxr-xr-x, got %s", dirName, info.Mode()) + } + + err = gcsAfs.RemoveAll(filepath.Join(bucketName, "a")) + if err != nil { + t.Fatalf("failed to remove the folder %s with error: %s", filepath.Join(bucketName, "a"), err) + } + }) } From c1220175da23ab98314f189d211cf1d20328c94c Mon Sep 17 00:00:00 2001 From: VOvchinnikov Date: Thu, 5 Aug 2021 18:38:36 +0200 Subject: [PATCH 4/6] Added implementation to cover the new `Mkdir` & `MkdirAll` cases --- gcsfs/fs.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/gcsfs/fs.go b/gcsfs/fs.go index ca452184..c4d899c7 100644 --- a/gcsfs/fs.go +++ b/gcsfs/fs.go @@ -161,6 +161,15 @@ func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error { if err := validateName(name); err != nil { return err } + // folder creation logic has to additionally check for folder name presence + bucketName, path := fs.splitName(name) + if bucketName == "" { + return ErrNoBucketInName + } + if path == "" { + // the API would throw "googleapi: Error 400: No object name, required", but this one is more consistent + return ErrEmptyObjectName + } obj, err := fs.getObj(name) if err != nil { @@ -175,6 +184,15 @@ func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error { if err := validateName(path); err != nil { return err } + // folder creation logic has to additionally check for folder name presence + bucketName, splitPath := fs.splitName(path) + if bucketName == "" { + return ErrNoBucketInName + } + if splitPath == "" { + // the API would throw "googleapi: Error 400: No object name, required", but this one is more consistent + return ErrEmptyObjectName + } root := "" folders := strings.Split(path, fs.separator) @@ -186,7 +204,9 @@ func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error { if root != "" { root = root + fs.separator + f } else { + // we have to have at least bucket name + folder name to create successfully root = f + continue } if err := fs.Mkdir(root, perm); err != nil { From 090e652a76bba9e787474d585bb5897a7c18d044 Mon Sep 17 00:00:00 2001 From: Martin Karlsch Date: Tue, 5 Oct 2021 18:17:13 +0200 Subject: [PATCH 5/6] read GOOGLE_APPLICATION_CREDENTIALS_JSON if it is set --- gcs.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gcs.go b/gcs.go index fd1f7175..11358221 100644 --- a/gcs.go +++ b/gcs.go @@ -37,6 +37,9 @@ type GcsFs struct { // You can provide additional options to be passed to the client creation, as per // cloud.google.com/go/storage documentation func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) { + if json := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON"); json != "" { + opts = append(opts, option.WithCredentialsJSON([]byte(json))) + } client, err := storage.NewClient(ctx, opts...) if err != nil { return nil, err From 68b71565dda3aee6ae797fc69ce5a16cda22967a Mon Sep 17 00:00:00 2001 From: Michalis Kargakis Date: Wed, 22 Dec 2021 11:03:34 +0100 Subject: [PATCH 6/6] Add README section for GCS Fs --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 0262ead4..cab257f5 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,18 @@ system using InMemoryFile. Afero has experimental support for secure file transfer protocol (sftp). Which can be used to perform file operations over a encrypted channel. +### GCSFs + +Afero has experimental support for Google Cloud Storage (GCS). You can either set the +`GOOGLE_APPLICATION_CREDENTIALS_JSON` env variable to your JSON credentials or use `opts` in +`NewGcsFS` to configure access to your GCS bucket. + +Some known limitations of the existing implementation: +* No Chmod support - The GCS ACL could probably be mapped to *nix style permissions but that would add another level of complexity and is ignored in this version. +* No Chtimes support - Could be simulated with attributes (gcs a/m-times are set implicitly) but that's is left for another version. +* Not thread safe - Also assumes all file operations are done through the same instance of the GcsFs. File operations between different GcsFs instances are not guaranteed to be consistent. + + ## Filtering Backends ### BasePathFs