Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add new filtering fs: FilePredicateFs #309

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
21 changes: 19 additions & 2 deletions README.md
Expand Up @@ -281,7 +281,7 @@ _, err := fs.Create("/file.txt")
// err = syscall.EPERM
```

# RegexpFs
### RegexpFs

A filtered view on file names, any file NOT matching
the passed regexp will be treated as non-existing.
Expand All @@ -294,7 +294,24 @@ _, err := fs.Create("/file.html")
// err = syscall.ENOENT
```

### HttpFs
### FilePredicateFs

A filtered view on predicates which takes file path as an argument,
any file will be treated as non-existing when the predicate returns false.
Unlike `RegexpFs`, the fs targets not file name but file path.
Like, `RegexpFs`, files will not be created when the predicate returns false
and directories are always not filtered.

```go
pred := func(path string) bool {
return strings.HasSuffix(path, ".txt")
}
fs := afero.NewFilePredicateFs(afero.NewMemMapFs(), pred)
_, err := fs.Create("/file.html")
// err = syscall.ENOENT
```

## HttpFs

Afero provides an http compatible backend which can wrap any of the existing
backends.
Expand Down
219 changes: 219 additions & 0 deletions predicatefs.go
@@ -0,0 +1,219 @@
package afero

import (
"os"
"path/filepath"
"syscall"
"time"
)

// FilePredicateFs filters files (not directories) by predicate,
// which takes file path as an arg.
type FilePredicateFs struct {
pred func(string) bool
source Fs
}

func NewFilePredicateFs(source Fs, pred func(string) bool) Fs {
return &FilePredicateFs{source: source, pred: pred}
}

type PredicateFile struct {
f File
pred func(string) bool
}

func (p *FilePredicateFs) validate(path string) error {
if p.pred(path) {
return nil
}
return syscall.ENOENT
}

func (p *FilePredicateFs) dirOrValidPath(path string) error {
dir, err := IsDir(p.source, path)
if err != nil {
return err
}
if dir {
return nil
}
return p.validate(path)
}

func (p *FilePredicateFs) Chtimes(path string, a, m time.Time) error {
if err := p.dirOrValidPath(path); err != nil {
return err
}
return p.source.Chtimes(path, a, m)
}

func (p *FilePredicateFs) Chmod(path string, mode os.FileMode) error {
if err := p.dirOrValidPath(path); err != nil {
return err
}
return p.source.Chmod(path, mode)
}

func (p *FilePredicateFs) Chown(path string, uid, gid int) error {
if err := p.dirOrValidPath(path); err != nil {
return err
}
return p.source.Chown(path, uid, gid)
}

func (p *FilePredicateFs) Name() string {
return "FilePredicateFs"
}

func (p *FilePredicateFs) Stat(path string) (os.FileInfo, error) {
if err := p.dirOrValidPath(path); err != nil {
return nil, err
}
return p.source.Stat(path)
}

func (p *FilePredicateFs) Rename(oldname, newname string) error {
dir, err := IsDir(p.source, oldname)
if err != nil {
return err
}
if dir {
return nil
}
if err := p.validate(oldname); err != nil {
return err
}
if err := p.validate(newname); err != nil {
return err
}
return p.source.Rename(oldname, newname)
}

func (p *FilePredicateFs) RemoveAll(path string) error {
dir, err := IsDir(p.source, path)
if err != nil {
return err
}
if !dir {
if err := p.validate(path); err != nil {
return err
}
}
return p.source.RemoveAll(path)
}

func (p *FilePredicateFs) Remove(path string) error {
if err := p.dirOrValidPath(path); err != nil {
return err
}
return p.source.Remove(path)
}

func (p *FilePredicateFs) OpenFile(path string, flag int, perm os.FileMode) (File, error) {
if err := p.dirOrValidPath(path); err != nil {
return nil, err
}
return p.source.OpenFile(path, flag, perm)
}

func (p *FilePredicateFs) Open(path string) (File, error) {
dir, err := IsDir(p.source, path)
if err != nil {
return nil, err
}
if !dir {
if err := p.validate(path); err != nil {
return nil, err
}
}
f, err := p.source.Open(path)
if err != nil {
return nil, err
}
return &PredicateFile{f: f, pred: p.pred}, nil
}

func (p *FilePredicateFs) Mkdir(n string, path os.FileMode) error {
return p.source.Mkdir(n, path)
}

func (p *FilePredicateFs) MkdirAll(n string, path os.FileMode) error {
return p.source.MkdirAll(n, path)
}

func (p *FilePredicateFs) Create(path string) (File, error) {
if err := p.validate(path); err != nil {
return nil, err
}
return p.source.Create(path)
}

func (f *PredicateFile) Close() error {
return f.f.Close()
}

func (f *PredicateFile) Read(s []byte) (int, error) {
return f.f.Read(s)
}

func (f *PredicateFile) ReadAt(s []byte, o int64) (int, error) {
return f.f.ReadAt(s, o)
}

func (f *PredicateFile) Seek(o int64, w int) (int64, error) {
return f.f.Seek(o, w)
}

func (f *PredicateFile) Write(s []byte) (int, error) {
return f.f.Write(s)
}

func (f *PredicateFile) WriteAt(s []byte, o int64) (int, error) {
return f.f.WriteAt(s, o)
}

func (f *PredicateFile) Name() string {
return f.f.Name()
}

func (f *PredicateFile) Readdir(c int) (fi []os.FileInfo, err error) {
var pfi []os.FileInfo
pfi, err = f.f.Readdir(c)
if err != nil {
return nil, err
}
for _, i := range pfi {
if i.IsDir() || f.pred(filepath.Join(f.f.Name(), i.Name())) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@satotake I'm not sure what you mean by "skip directories"; to me it looks like every directory is included.

I would have found it much more useful if the above was replaced with:

if f.pred(i.IsDir(), filepath.Join(f.f.Name(), i.Name())) {

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One might as well just pass the file info object instead of a bool which can then be manually checked whether the path is a directory or a file or a symlink, etc.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That would break if you want to use this with fs.ReadDirFile (see my recent commit to the main branch).

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@bep I got your point.
isDir will be nice to ignore some dirs without scanning their children, for instance.
Fixed.

fi = append(fi, i)
}
}
return fi, nil
}

func (f *PredicateFile) Readdirnames(c int) (n []string, err error) {
fi, err := f.Readdir(c)
if err != nil {
return nil, err
}
for _, s := range fi {
n = append(n, s.Name())
}
return n, nil
}

func (f *PredicateFile) Stat() (os.FileInfo, error) {
return f.f.Stat()
}

func (f *PredicateFile) Sync() error {
return f.f.Sync()
}

func (f *PredicateFile) Truncate(s int64) error {
return f.f.Truncate(s)
}

func (f *PredicateFile) WriteString(s string) (int, error) {
return f.f.WriteString(s)
}
65 changes: 65 additions & 0 deletions predicatefs_test.go
@@ -0,0 +1,65 @@
package afero

import (
"path/filepath"
"strings"
"testing"
)

func TestFilePredicateFs(t *testing.T) {
mfs := &MemMapFs{}

txtExts := func(name string) bool {
return strings.HasSuffix(name, ".txt")
}

nonEmpty := func(name string) bool {
fi, err := mfs.Stat(name)
if err != nil {
t.Errorf("Got unexpected Stat err %v", err)
}
// Note: If you use this rule, you cannot create any files on the fs
return fi.Size() > 0
}

inHiddenDir := func(path string) bool {
return strings.HasSuffix(filepath.Dir(path), ".hidden")
}

pred := func(path string) bool {
return nonEmpty(path) && txtExts(path) && !inHiddenDir(path)
}

fs := &FilePredicateFs{pred: pred, source: mfs}

mfs.MkdirAll("/dir/sub/.hidden", 0777)
for _, name := range []string{"file.txt", "file.html", "empty.txt"} {
for _, dir := range []string{"/dir/", "/dir/sub/", "/dir/sub/.hidden/"} {
fh, _ := mfs.Create(dir + name)

if !strings.HasPrefix(name, "empty") {
fh.WriteString("file content")
}

fh.Close()
}
}

files, _ := ReadDir(fs, "/dir")

if len(files) != 2 { // file.txt, sub
t.Errorf("Got wrong number of files: %#v", files)
}

f, _ := fs.Open("/dir/sub")
names, _ := f.Readdirnames(-1)
if len(names) != 2 {
// file.txt, .hidden (dirs are not filtered)
t.Errorf("Got wrong number of names: %v", names)
}

hiddenFiles, _ := ReadDir(fs, "/dir/sub/.hidden")
if len(hiddenFiles) != 0 {
t.Errorf("Got wrong number of names: %v", hiddenFiles)
}
}