Add GCS Fs implementation

gcs-fake-service-account.json

@@ -0,0 +1,9 @@
+{
+  "type": "service_account",
+  "private_key_id": "abc",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDY3E8o1NEFcjMM\nHW/5ZfFJw29/8NEqpViNjQIx95Xx5KDtJ+nWn9+OW0uqsSqKlKGhAdAo+Q6bjx2c\nuXVsXTu7XrZUY5Kltvj94DvUa1wjNXs606r/RxWTJ58bfdC+gLLxBfGnB6CwK0YQ\nxnfpjNbkUfVVzO0MQD7UP0Hl5ZcY0Puvxd/yHuONQn/rIAieTHH1pqgW+zrH/y3c\n59IGThC9PPtugI9ea8RSnVj3PWz1bX2UkCDpy9IRh9LzJLaYYX9RUd7++dULUlat\nAaXBh1U6emUDzhrIsgApjDVtimOPbmQWmX1S60mqQikRpVYZ8u+NDD+LNw+/Eovn\nxCj2Y3z1AgMBAAECggEAWDBzoqO1IvVXjBA2lqId10T6hXmN3j1ifyH+aAqK+FVl\nGjyWjDj0xWQcJ9ync7bQ6fSeTeNGzP0M6kzDU1+w6FgyZqwdmXWI2VmEizRjwk+/\n/uLQUcL7I55Dxn7KUoZs/rZPmQDxmGLoue60Gg6z3yLzVcKiDc7cnhzhdBgDc8vd\nQorNAlqGPRnm3EqKQ6VQp6fyQmCAxrr45kspRXNLddat3AMsuqImDkqGKBmF3Q1y\nxWGe81LphUiRqvqbyUlh6cdSZ8pLBpc9m0c3qWPKs9paqBIvgUPlvOZMqec6x4S6\nChbdkkTRLnbsRr0Yg/nDeEPlkhRBhasXpxpMUBgPywKBgQDs2axNkFjbU94uXvd5\nznUhDVxPFBuxyUHtsJNqW4p/ujLNimGet5E/YthCnQeC2P3Ym7c3fiz68amM6hiA\nOnW7HYPZ+jKFnefpAtjyOOs46AkftEg07T9XjwWNPt8+8l0DYawPoJgbM5iE0L2O\nx8TU1Vs4mXc+ql9F90GzI0x3VwKBgQDqZOOqWw3hTnNT07Ixqnmd3dugV9S7eW6o\nU9OoUgJB4rYTpG+yFqNqbRT8bkx37iKBMEReppqonOqGm4wtuRR6LSLlgcIU9Iwx\nyfH12UWqVmFSHsgZFqM/cK3wGev38h1WBIOx3/djKn7BdlKVh8kWyx6uC8bmV+E6\nOoK0vJD6kwKBgHAySOnROBZlqzkiKW8c+uU2VATtzJSydrWm0J4wUPJifNBa/hVW\ndcqmAzXC9xznt5AVa3wxHBOfyKaE+ig8CSsjNyNZ3vbmr0X04FoV1m91k2TeXNod\njMTobkPThaNm4eLJMN2SQJuaHGTGERWC0l3T18t+/zrDMDCPiSLX1NAvAoGBAN1T\nVLJYdjvIMxf1bm59VYcepbK7HLHFkRq6xMJMZbtG0ryraZjUzYvB4q4VjHk2UDiC\nlhx13tXWDZH7MJtABzjyg+AI7XWSEQs2cBXACos0M4Myc6lU+eL+iA+OuoUOhmrh\nqmT8YYGu76/IBWUSqWuvcpHPpwl7871i4Ga/I3qnAoGBANNkKAcMoeAbJQK7a/Rn\nwPEJB+dPgNDIaboAsh1nZhVhN5cvdvCWuEYgOGCPQLYQF0zmTLcM+sVxOYgfy8mV\nfbNgPgsP5xmu6dw2COBKdtozw0HrWSRjACd1N4yGu75+wPCcX/gQarcjRcXXZeEa\nNtBLSfcqPULqD+h7br9lEJio\n-----END PRIVATE KEY-----\n",
+  "client_email": "123-abc@developer.gserviceaccount.com",
+  "client_id": "123-abc.apps.googleusercontent.com",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "http://localhost:8080/token"
+}

gcs.go

@@ -0,0 +1,113 @@
+// Copyright © 2021 Vasily Ovchinnikov <vasily@remerge.io>.
+//
+// The code in this file is derived from afero fork github.com/Zatte/afero by Mikael Rapp
+// licensed under Apache License 2.0.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package afero
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"github.com/googleapis/google-cloud-go-testing/storage/stiface"
+	"github.com/spf13/afero/gcsfs"
+	"google.golang.org/api/option"
+)
+
+type GcsFs struct {
+	source *gcsfs.GcsFs
+}
+
+// Creates a GCS file system, automatically instantiating and decorating the storage client.
+// You can provide additional options to be passed to the client creation, as per
+// cloud.google.com/go/storage documentation
+func NewGcsFS(ctx context.Context, bucketName string, opts ...option.ClientOption) (Fs, error) {
+	client, err := storage.NewClient(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewGcsFSFromClient(ctx, client, bucketName)
+}
+
+// The same as NewGcsFS, but the files system will use the provided folder separator.
+func NewGcsFSWithSeparator(ctx context.Context, bucketName, folderSeparator string, opts ...option.ClientOption) (Fs, error) {
+	client, err := storage.NewClient(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewGcsFSFromClientWithSeparator(ctx, client, bucketName, folderSeparator)
+}
+
+// Creates a GCS file system from a given storage client
+func NewGcsFSFromClient(ctx context.Context, client *storage.Client, bucketName string) (Fs, error) {
+	c := stiface.AdaptClient(client)
+
+	bucket := c.Bucket(bucketName)
+
+	return &GcsFs{gcsfs.NewGcsFs(ctx, bucket)}, nil
+}
+
+// Same as NewGcsFSFromClient, but the file system will use the provided folder separator.
+func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, bucketName, folderSeparator string) (Fs, error) {
+	c := stiface.AdaptClient(client)
+
+	bucket := c.Bucket(bucketName)
+
+	return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, bucket, folderSeparator)}, nil
+}
+
+// Wraps gcs.GcsFs and convert some return types to afero interfaces.
+func (fs *GcsFs) Name() string {
+	return fs.source.Name()
+}
+func (fs *GcsFs) Create(name string) (File, error) {
+	return fs.source.Create(name)
+}
+func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
+	return fs.source.Mkdir(name, perm)
+}
+func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
+	return fs.source.MkdirAll(path, perm)
+}
+func (fs *GcsFs) Open(name string) (File, error) {
+	return fs.source.Open(name)
+}
+func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
+	return fs.source.OpenFile(name, flag, perm)
+}
+func (fs *GcsFs) Remove(name string) error {
+	return fs.source.Remove(name)
+}
+func (fs *GcsFs) RemoveAll(path string) error {
+	return fs.source.RemoveAll(path)
+}
+func (fs *GcsFs) Rename(oldname, newname string) error {
+	return fs.source.Rename(oldname, newname)
+}
+func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
+	return fs.source.Stat(name)
+}
+func (fs *GcsFs) Chmod(name string, mode os.FileMode) error {
+	return fs.source.Chmod(name, mode)
+}
+func (fs *GcsFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	return fs.source.Chtimes(name, atime, mtime)
+}
+func (fs *GcsFs) Chown(name string, uid, gid int) error {
+	return fs.source.Chown(name, uid, gid)
+}

gcs_mocks.go

@@ -0,0 +1,255 @@
+// Copyright © 2021 Vasily Ovchinnikov <vasily@remerge.io>.
+//
+// A set of stiface-based mocks, replicating the GCS behavior, to make the tests not require any
+// internet connection or real buckets.
+// It is **not** a comprehensive set of mocks to test anything and everything GCS-related, rather
+// a very tailored one for the current implementation - thus the tests, written with the use of
+// these mocks are more of regression ones.
+// If any GCS behavior changes and breaks the implementation, then it should first be adjusted by
+// switching over to a real bucket - and then the mocks have to be adjusted to match the
+// implementation.
+
+package afero
+
+import (
+	"context"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/spf13/afero/gcsfs"
+
+	"cloud.google.com/go/storage"
+	"github.com/googleapis/google-cloud-go-testing/storage/stiface"
+	"google.golang.org/api/iterator"
+)
+
+// sets filesystem separators to the one, expected (and hard-coded) in the tests
+func normSeparators(s string) string {
+	return strings.Replace(s, "\\", "/", -1)
+}
+
+type clientMock struct {
+	stiface.Client
+	fs Fs
+}
+
+func newClientMock() *clientMock {
+	return &clientMock{fs: NewMemMapFs()}
+}
+
+func (m *clientMock) Bucket(name string) stiface.BucketHandle {
+	return &bucketMock{bucketName: name, fs: m.fs}
+}
+
+type bucketMock struct {
+	stiface.BucketHandle
+
+	bucketName string
+
+	fs Fs
+}
+
+func (m *bucketMock) Object(name string) stiface.ObjectHandle {
+	return &objectMock{name: name, fs: m.fs}
+}
+
+func (m *bucketMock) Objects(_ context.Context, q *storage.Query) (it stiface.ObjectIterator) {
+	return &objectItMock{name: q.Prefix, fs: m.fs}
+}
+
+type objectMock struct {
+	stiface.ObjectHandle
+
+	name string
+	fs   Fs
+}
+
+func (o *objectMock) NewWriter(_ context.Context) stiface.Writer {
+	return &writerMock{name: o.name, fs: o.fs}
+}
+
+func (o *objectMock) NewRangeReader(_ context.Context, offset, length int64) (stiface.Reader, error) {
+	if o.name == "" {
+		return nil, gcsfs.ErrEmptyObjectName
+	}
+
+	file, err := o.fs.Open(o.name)
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = file.Seek(offset, io.SeekStart)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	res := &readerMock{file: file}
+	if length > -1 {
+		res.buf = make([]byte, length)
+		_, err = file.Read(res.buf)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return res, nil
+}
+
+func (o *objectMock) Delete(_ context.Context) error {
+	if o.name == "" {
+		return gcsfs.ErrEmptyObjectName
+	}
+	return o.fs.Remove(o.name)
+}
+
+func (o *objectMock) Attrs(_ context.Context) (*storage.ObjectAttrs, error) {
+	if o.name == "" {
+		return nil, gcsfs.ErrEmptyObjectName
+	}
+
+	info, err := o.fs.Stat(o.name)
+	if err != nil {
+		pathError, ok := err.(*os.PathError)
+		if ok {
+			if pathError.Err == os.ErrNotExist {
+				return nil, storage.ErrObjectNotExist
+			}
+		}
+
+		return nil, err
+	}
+
+	res := &storage.ObjectAttrs{Name: normSeparators(o.name), Size: info.Size(), Updated: info.ModTime()}
+
+	if info.IsDir() {
+		// we have to mock it here, because of FileInfo logic
+		return nil, gcsfs.ErrObjectDoesNotExist
+	}
+
+	return res, nil
+}
+
+type writerMock struct {
+	stiface.Writer
+
+	name string
+	fs   Fs
+
+	file File
+}
+
+func (w *writerMock) Write(p []byte) (n int, err error) {
+	if w.file == nil {
+		w.file, err = w.fs.Create(w.name)
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	return w.file.Write(p)
+}
+
+func (w *writerMock) Close() error {
+	if w.file == nil {
+		var err error
+		if strings.HasSuffix(w.name, "/") {
+			err = w.fs.Mkdir(w.name, 0755)
+			if err != nil {
+				return err
+			}
+		} else {
+			_, err = w.Write([]byte{})
+			if err != nil {
+				return err
+			}
+		}
+	}
+	if w.file != nil {
+		return w.file.Close()
+	}
+	return nil
+}
+
+type readerMock struct {
+	stiface.Reader
+
+	file File
+
+	buf []byte
+}
+
+func (r *readerMock) Remain() int64 {
+	return 0
+}
+
+func (r *readerMock) Read(p []byte) (int, error) {
+	if r.buf != nil {
+		copy(p, r.buf)
+		return len(r.buf), nil
+	}
+	return r.file.Read(p)
+}
+
+type objectItMock struct {
+	stiface.ObjectIterator
+
+	name string
+	fs   Fs
+
+	dir   File
+	infos []*storage.ObjectAttrs
+}
+
+func (it *objectItMock) Next() (*storage.ObjectAttrs, error) {
+	var err error
+	if it.dir == nil {
+		it.dir, err = it.fs.Open(it.name)
+		if err != nil {
+			return nil, err
+		}
+
+		var isDir bool
+		isDir, err = IsDir(it.fs, it.name)
+		if err != nil {
+			return nil, err
+		}
+
+		it.infos = []*storage.ObjectAttrs{}
+
+		if !isDir {
+			var info os.FileInfo
+			info, err = it.dir.Stat()
+			if err != nil {
+				return nil, err
+			}
+			it.infos = append(it.infos, &storage.ObjectAttrs{Name: normSeparators(info.Name()), Size: info.Size(), Updated: info.ModTime()})
+		} else {
+			var fInfos []os.FileInfo
+			fInfos, err = it.dir.Readdir(0)
+			if err != nil {
+				return nil, err
+			}
+			if it.name != "" {
+				it.infos = append(it.infos, &storage.ObjectAttrs{
+					Prefix: normSeparators(it.name) + "/",
+				})
+			}
+
+			for _, info := range fInfos {
+				it.infos = append(it.infos, &storage.ObjectAttrs{Name: normSeparators(info.Name()), Size: info.Size(), Updated: info.ModTime()})
+			}
+		}
+	}
+
+	if len(it.infos) == 0 {
+		return nil, iterator.Done
+	}
+
+	res := it.infos[0]
+	it.infos = it.infos[1:]
+
+	return res, err
+}