From a0e528ef89eedbd3dd88304b4655c71e5094f0a9 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Fri, 23 Dec 2022 11:22:33 +0900 Subject: [PATCH 1/3] Do not escape the object name in the fs backend Now fake-gcs-server implicitly creates parent directories when creating an object /foo/bar/baz. This allows creating paths longer than NAME_MAX (256) and interoperating with existing filesystems. This approach does not allow some kinds of objects to exist, e.g., /foo/ with a non-zero object size, although these should be rare. --- internal/backend/fs.go | 44 +++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/internal/backend/fs.go b/internal/backend/fs.go index 01f1ccef7d..8cce7d9391 100644 --- a/internal/backend/fs.go +++ b/internal/backend/fs.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "io/fs" "net/url" "os" "path/filepath" @@ -178,7 +179,16 @@ func (s *storageFS) CreateObject(obj StreamingObject, conditions Conditions) (St return StreamingObject{}, PreConditionFailed } - path := filepath.Join(s.rootDir, url.PathEscape(obj.BucketName), url.PathEscape(obj.Name)) + path := filepath.Join(s.rootDir, url.PathEscape(obj.BucketName), obj.Name) + if err = os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return StreamingObject{}, err + } + + // Nothing to do if this operation only creates directories + if strings.HasSuffix(obj.Name, "/") { + // TODO: populate Crc32c, Md5Hash, and Etag + return StreamingObject{obj.ObjectAttrs, noopSeekCloser{bytes.NewReader([]byte{})}}, nil + } var buf bytes.Buffer hasher := checksum.NewStreamingHasher() @@ -217,28 +227,30 @@ func (s *storageFS) ListObjects(bucketName string, prefix string, versions bool) s.mtx.RLock() defer s.mtx.RUnlock() - infos, err := os.ReadDir(filepath.Join(s.rootDir, url.PathEscape(bucketName))) - if err != nil { - return nil, err - } objects := []ObjectAttrs{} - for _, info := range infos { + // TODO: WalkDir more efficient? + bucketPath := filepath.Join(s.rootDir, url.PathEscape(bucketName)) + if err := filepath.Walk(bucketPath, func(path string, info fs.FileInfo, _ error) error { + objName := strings.TrimPrefix(path, bucketPath+string(filepath.Separator)) + // TODO: should this check path? if s.mh.isSpecialFile(info.Name()) { - continue + return nil } - unescaped, err := url.PathUnescape(info.Name()) - if err != nil { - return nil, fmt.Errorf("failed to unescape object name %s: %w", info.Name(), err) + if info.IsDir() { + return nil } - if prefix != "" && !strings.HasPrefix(unescaped, prefix) { - continue + if prefix != "" && !strings.HasPrefix(objName, prefix) { + return nil } - object, err := s.getObject(bucketName, unescaped) + object, err := s.getObject(bucketName, objName) if err != nil { - return nil, err + return err } object.Close() objects = append(objects, object.ObjectAttrs) + return nil + }); err != nil { + return nil, err } return objects, nil } @@ -264,7 +276,7 @@ func (s *storageFS) GetObjectWithGeneration(bucketName, objectName string, gener } func (s *storageFS) getObject(bucketName, objectName string) (StreamingObject, error) { - path := filepath.Join(s.rootDir, url.PathEscape(bucketName), url.PathEscape(objectName)) + path := filepath.Join(s.rootDir, url.PathEscape(bucketName), objectName) encoded, err := s.mh.read(path) if err != nil { @@ -303,7 +315,7 @@ func (s *storageFS) DeleteObject(bucketName, objectName string) error { if objectName == "" { return errors.New("can't delete object with empty name") } - path := filepath.Join(s.rootDir, url.PathEscape(bucketName), url.PathEscape(objectName)) + path := filepath.Join(s.rootDir, url.PathEscape(bucketName), objectName) if err := s.mh.remove(path); err != nil { return err } From 89ff764e850f7e7ac8ceab9ead71400cde7af53b Mon Sep 17 00:00:00 2001 From: fsouza <108725+fsouza@users.noreply.github.com> Date: Wed, 28 Dec 2022 17:45:10 -0500 Subject: [PATCH 2/3] Use filepath.Rel to derive the object name from the path --- internal/backend/fs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backend/fs.go b/internal/backend/fs.go index 8cce7d9391..18c40a462d 100644 --- a/internal/backend/fs.go +++ b/internal/backend/fs.go @@ -231,7 +231,7 @@ func (s *storageFS) ListObjects(bucketName string, prefix string, versions bool) // TODO: WalkDir more efficient? bucketPath := filepath.Join(s.rootDir, url.PathEscape(bucketName)) if err := filepath.Walk(bucketPath, func(path string, info fs.FileInfo, _ error) error { - objName := strings.TrimPrefix(path, bucketPath+string(filepath.Separator)) + objName := filepath.Rel(bucketPath, path) // TODO: should this check path? if s.mh.isSpecialFile(info.Name()) { return nil From 654b0a7c8203a728093112babac7917ff0ffec17 Mon Sep 17 00:00:00 2001 From: francisco souza <108725+fsouza@users.noreply.github.com> Date: Wed, 28 Dec 2022 17:49:14 -0500 Subject: [PATCH 3/3] internal/backend/fs: fix usage of filepath.Rel --- internal/backend/fs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/backend/fs.go b/internal/backend/fs.go index 18c40a462d..27a3a5f003 100644 --- a/internal/backend/fs.go +++ b/internal/backend/fs.go @@ -231,7 +231,8 @@ func (s *storageFS) ListObjects(bucketName string, prefix string, versions bool) // TODO: WalkDir more efficient? bucketPath := filepath.Join(s.rootDir, url.PathEscape(bucketName)) if err := filepath.Walk(bucketPath, func(path string, info fs.FileInfo, _ error) error { - objName := filepath.Rel(bucketPath, path) + // Rel() should never return error since path always descend from bucketPath + objName, _ := filepath.Rel(bucketPath, path) // TODO: should this check path? if s.mh.isSpecialFile(info.Name()) { return nil