diff --git a/fsnotify.go b/fsnotify.go index 4ff4e3db..07f70f48 100644 --- a/fsnotify.go +++ b/fsnotify.go @@ -12,9 +12,28 @@ package fsnotify import ( "errors" "fmt" + "io/fs" + "path/filepath" "strings" ) +// These are the generalized file operations that can trigger a notification. +const ( + Create Op = 1 << iota + Write + Remove + Rename + Chmod +) + +// Common errors that can be reported by a watcher +var ( + ErrNonExistentWatch = errors.New("can't remove non-existent watcher") + ErrEventOverflow = errors.New("fsnotify queue overflow") + ErrNotDirectory = errors.New("not a directory") + ErrRecursionUnsupported = errors.New("recursion not supported") +) + // Event represents a single file system notification. type Event struct { // Path to the file or directory. @@ -34,21 +53,6 @@ type Event struct { // Op describes a set of file operations. type Op uint32 -// These are the generalized file operations that can trigger a notification. -const ( - Create Op = 1 << iota - Write - Remove - Rename - Chmod -) - -// Common errors that can be reported by a watcher -var ( - ErrNonExistentWatch = errors.New("can't remove non-existent watcher") - ErrEventOverflow = errors.New("fsnotify queue overflow") -) - func (op Op) String() string { var b strings.Builder if op.Has(Create) { @@ -83,3 +87,34 @@ func (e Event) Has(op Op) bool { return e.Op.Has(op) } func (e Event) String() string { return fmt.Sprintf("%q: %s", e.Name, e.Op.String()) } + +// findDirs finds all directories under path (return value *includes* path as +// the first entry). +func findDirs(path string) ([]string, error) { + dirs := make([]string, 0, 8) + err := filepath.WalkDir(path, func(root string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if root == path && !d.IsDir() { + return fmt.Errorf("%q: %w", path, ErrNotDirectory) + } + if d.IsDir() { + dirs = append(dirs, root) + } + return nil + }) + if err != nil { + return nil, err + } + return dirs, nil +} + +// Check if this path is recursive (ends with "/..."), and return the path with +// the /... stripped. +func recursivePath(path string) (string, bool) { + if filepath.Base(path) == "..." { + return filepath.Dir(path), true + } + return path, false +} diff --git a/fsnotify_test.go b/fsnotify_test.go index 5f6c8827..444e57be 100644 --- a/fsnotify_test.go +++ b/fsnotify_test.go @@ -4,6 +4,8 @@ package fsnotify import ( + "path/filepath" + "strings" "testing" ) @@ -34,3 +36,48 @@ func TestEventString(t *testing.T) { }) } } + +func TestFindDirs(t *testing.T) { + join := func(list ...string) string { + return "\n\t" + strings.Join(list, "\n\t") + } + + t.Run("finds dirs", func(t *testing.T) { + tmp := t.TempDir() + + mkdirAll(t, tmp, "/one/two/three/four") + cat(t, "asd", tmp, "one/two/file.txt") + symlink(t, "/", tmp, "link") + + dirs, err := findDirs(tmp) + if err != nil { + t.Fatal(err) + } + + have := join(dirs...) + want := join([]string{ + tmp, + filepath.Join(tmp, "one"), + filepath.Join(tmp, "one/two"), + filepath.Join(tmp, "one/two/three"), + filepath.Join(tmp, "one/two/three/four"), + }...) + + if have != want { + t.Errorf("\nhave: %s\nwant: %s", have, want) + } + }) + + t.Run("file", func(t *testing.T) { + tmp := t.TempDir() + cat(t, "asd", tmp, "file") + + dirs, err := findDirs(filepath.Join(tmp, "file")) + if !errorContains(err, "not a directory") { + t.Errorf("wrong error: %s", err) + } + if len(dirs) > 0 { + t.Errorf("dirs contains entries: %s", dirs) + } + }) +} diff --git a/helpers_test.go b/helpers_test.go index 08164fd0..8cc1abd1 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -95,19 +95,19 @@ func mkdir(t *testing.T, path ...string) { } // mkdir -p -// func mkdirAll(t *testing.T, path ...string) { -// t.Helper() -// if len(path) < 1 { -// t.Fatalf("mkdirAll: path must have at least one element: %s", path) -// } -// err := os.MkdirAll(filepath.Join(path...), 0o0755) -// if err != nil { -// t.Fatalf("mkdirAll(%q): %s", filepath.Join(path...), err) -// } -// if shouldWait(path...) { -// eventSeparator() -// } -// } +func mkdirAll(t *testing.T, path ...string) { + t.Helper() + if len(path) < 1 { + t.Fatalf("mkdirAll: path must have at least one element: %s", path) + } + err := os.MkdirAll(filepath.Join(path...), 0o0755) + if err != nil { + t.Fatalf("mkdirAll(%q): %s", filepath.Join(path...), err) + } + if shouldWait(path...) { + eventSeparator() + } +} // ln -s func symlink(t *testing.T, target string, link ...string) { @@ -441,3 +441,13 @@ func cmpEvents(t *testing.T, tmp string, have, want Events) { func indent(s fmt.Stringer) string { return "\t" + strings.ReplaceAll(s.String(), "\n", "\n\t") } + +func errorContains(out error, want string) bool { + if out == nil { + return want == "" + } + if want == "" { + return false + } + return strings.Contains(out.Error(), want) +} diff --git a/inotify.go b/inotify.go index 454e81e2..668ff7eb 100644 --- a/inotify.go +++ b/inotify.go @@ -30,11 +30,16 @@ type Watcher struct { mu sync.Mutex // Map access inotifyFile *os.File watches map[string]*watch // Map of inotify watches (key: path) - paths map[int]string // Map of watched paths (key: watch descriptor) + paths map[int]watchPath // Map of watched paths (key: watch descriptor) done chan struct{} // Channel for sending a "quit message" to the reader goroutine doneResp chan struct{} // Channel to respond to Close } +type watchPath struct { + path string + recurse bool +} + // NewWatcher establishes a new watcher with the underlying OS and begins waiting for events. func NewWatcher() (*Watcher, error) { // Create inotify fd @@ -49,7 +54,7 @@ func NewWatcher() (*Watcher, error) { fd: fd, inotifyFile: os.NewFile(uintptr(fd), ""), watches: make(map[string]*watch), - paths: make(map[int]string), + paths: make(map[int]watchPath), Events: make(chan Event), Errors: make(chan error), done: make(chan struct{}), @@ -93,33 +98,56 @@ func (w *Watcher) Close() error { return nil } -// Add starts watching the named file or directory (non-recursively). -func (w *Watcher) Add(name string) error { - name = filepath.Clean(name) +// Add starts watching a file or directory. +// +// If the path is a directory then changes to that directory are watched +// non-recursively. If the path ends with "..." changes in the entire directory +// tree are watched. ErrNotDirectory is returned when using "..." on a file. +// +// Symlinks are not followed. +func (w *Watcher) Add(path string) error { + path = filepath.Clean(path) if w.isClosed() { return errors.New("inotify instance already closed") } - const agnosticEvents = unix.IN_MOVED_TO | unix.IN_MOVED_FROM | + path, recurse := recursivePath(path) + if recurse { + dirs, err := findDirs(path) + if err != nil { + return err + } + for _, d := range dirs { + err := w.add(d, true) + if err != nil { + return err + } + } + return nil + } + + return w.add(path, false) +} + +func (w *Watcher) add(path string, recurse bool) error { + var flags uint32 = unix.IN_MOVED_TO | unix.IN_MOVED_FROM | unix.IN_CREATE | unix.IN_ATTRIB | unix.IN_MODIFY | unix.IN_MOVE_SELF | unix.IN_DELETE | unix.IN_DELETE_SELF - var flags uint32 = agnosticEvents - w.mu.Lock() defer w.mu.Unlock() - watchEntry := w.watches[name] + watchEntry := w.watches[path] if watchEntry != nil { flags |= watchEntry.flags | unix.IN_MASK_ADD } - wd, errno := unix.InotifyAddWatch(w.fd, name, flags) + wd, errno := unix.InotifyAddWatch(w.fd, path, flags) if wd == -1 { return errno } if watchEntry == nil { - w.watches[name] = &watch{wd: uint32(wd), flags: flags} - w.paths[wd] = name + w.watches[path] = &watch{wd: uint32(wd), flags: flags} + w.paths[wd] = watchPath{path: path, recurse: recurse} } else { watchEntry.wd = uint32(wd) watchEntry.flags = flags @@ -128,25 +156,32 @@ func (w *Watcher) Add(name string) error { return nil } -// Remove stops watching the named file or directory (non-recursively). -func (w *Watcher) Remove(name string) error { - name = filepath.Clean(name) +// Remove stops watching a file or directory. +// +// If a path was added recursively with "..." then it will stop watching the +// entire directory tree. You can optionally add "..."; in this case +// Remove("path") and Remove("path/...") behave identical. +// +// In other cases adding "..." will any watches in that tree (if any). +// ErrNotDirectory is returned when using "..." on a file. +func (w *Watcher) Remove(path string) error { + path = filepath.Clean(path) // Fetch the watch. w.mu.Lock() defer w.mu.Unlock() - watch, ok := w.watches[name] + watch, ok := w.watches[path] // Remove it from inotify. if !ok { - return fmt.Errorf("%w: %s", ErrNonExistentWatch, name) + return fmt.Errorf("%w: %s", ErrNonExistentWatch, path) } // We successfully removed the watch if InotifyRmWatch doesn't return an // error, we need to clean up our internal state to ensure it matches // inotify's kernel state. delete(w.paths, int(watch.wd)) - delete(w.watches, name) + delete(w.watches, path) // inotify_rm_watch will return EINVAL if the file has been deleted; // the inotify will already have been removed. @@ -260,7 +295,8 @@ func (w *Watcher) readEvents() { // the "Name" field with a valid filename. We retrieve the path of the watch from // the "paths" map. w.mu.Lock() - name, ok := w.paths[int(raw.Wd)] + watchPath, ok := w.paths[int(raw.Wd)] + name := watchPath.path // IN_DELETE_SELF occurs when the file/directory being watched is removed. // This is a sign to clean up the maps, otherwise we are no longer in sync // with the inotify kernel state which has already deleted the watch @@ -278,7 +314,7 @@ func (w *Watcher) readEvents() { name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") } - event := newEvent(name, mask) + event := w.newEvent(name, mask) // Send the events that are not ignored on the events channel if mask&unix.IN_IGNORED == 0 { @@ -295,13 +331,65 @@ func (w *Watcher) readEvents() { } } +// Certain types of events can be "ignored" and not sent over the Events +// channel. Such as events marked ignore by the kernel, or MODIFY events +// against files that do not exist. +func (e *Event) ignoreLinux(mask uint32) bool { + // Ignore anything the inotify API says to ignore + if mask&unix.IN_IGNORED == unix.IN_IGNORED { + return true + } + + // If the event is Create or Write, the file must exist, or the + // event will be suppressed. + // *Note*: this was put in place because it was seen that a Write + // event was sent after the Remove. This ignores the Write and + // assumes a Remove will come or has come if the file doesn't exist. + if e.Op&Create == Create || e.Op&Write == Write { + _, statErr := os.Lstat(e.Name) + return os.IsNotExist(statErr) + } + return false +} + +func (w *Watcher) isRecursive(path string) *watch { + ww, ok := w.watches[path] + if !ok { + path = filepath.Dir(path) + ww, ok = w.watches[path] + if !ok { + return nil + } + } + if !w.paths[int(ww.wd)].recurse { + return nil + } + return ww +} + // newEvent returns an platform-independent Event based on an inotify mask. -func newEvent(name string, mask uint32) Event { +func (w *Watcher) newEvent(name string, mask uint32) Event { e := Event{Name: name} if mask&unix.IN_CREATE == unix.IN_CREATE || mask&unix.IN_MOVED_TO == unix.IN_MOVED_TO { e.Op |= Create + + // Add new directories on recursive watches. + if mask&unix.IN_ISDIR == unix.IN_ISDIR { + ww := w.isRecursive(name) + if ww != nil { + //err := w.add(name, true) + err := w.Add(filepath.Join(name, "...")) + if err != nil { + // TODO: not sure if this has a nice error message. + // Also, this path could have been removed by now; + // should probably filter ENOENT or something. + w.Errors <- err + } + } + } } if mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF || mask&unix.IN_DELETE == unix.IN_DELETE { + // TODO: remove recursive watches. e.Op |= Remove } if mask&unix.IN_MODIFY == unix.IN_MODIFY { @@ -309,6 +397,20 @@ func newEvent(name string, mask uint32) Event { } if mask&unix.IN_MOVE_SELF == unix.IN_MOVE_SELF || mask&unix.IN_MOVED_FROM == unix.IN_MOVED_FROM { e.Op |= Rename + + if mask&unix.IN_ISDIR == unix.IN_ISDIR { + // TODO: should probably remove some things as well. + // ww := w.isRecursive(name) + // if ww != nil { + // err := w.Add(filepath.Join(name, "...")) + // if err != nil { + // // TODO: not sure if this has a nice error message. + // // Also, this path could have been removed by now; + // // should probably filter ENOENT or something. + // w.Errors <- err + // } + // } + } } if mask&unix.IN_ATTRIB == unix.IN_ATTRIB { e.Op |= Chmod diff --git a/integration_test.go b/integration_test.go index fff60748..7b5c6c2e 100644 --- a/integration_test.go +++ b/integration_test.go @@ -4,6 +4,7 @@ package fsnotify import ( + "errors" "fmt" "path/filepath" "runtime" @@ -523,3 +524,141 @@ func TestRemove(t *testing.T) { } }) } + +func TestRecursive(t *testing.T) { + switch runtime.GOOS { + case "linux": + // Run test. + default: + tmp := t.TempDir() + w := newWatcher(t) + err := w.Add(filepath.Join(tmp, "...")) + if !errors.Is(err, ErrRecursionUnsupported) { + t.Errorf("wrong error: %s", err) + } + return + } + + // inotify(7): + // Inotify monitoring of directories is not recursive: to monitor + // subdirectories under a directory, additional watches must be created. + // This can take a significant amount time for large directory trees. + // + // If monitoring an entire directory subtree, and a new subdirectory is + // created in that tree or an existing directory is renamed into that + // tree, be aware that by the time you create a watch for the new + // subdirectory, new files (and subdirectories) may already exist inside + // the subdirectory. Therefore, you may want to scan the contents of the + // subdirectory immediately after adding the watch (and, if desired, + // recursively add watches for any subdirectories that it contains). + + tests := []struct { + name string + preWatch func(*testing.T, string) + postWatch func(*testing.T, string) + want Events + }{ + {"basic", + func(t *testing.T, tmp string) { + mkdirAll(t, tmp, "/one/two/three/four") + }, + func(t *testing.T, tmp string) { + cat(t, "asd", tmp, "file.txt") + cat(t, "asd", tmp, "one/two/three/file.txt") + }, + Events{ + {"/file.txt", Create}, + {"/file.txt", Write}, + {"/one/two/three/file.txt", Create}, + {"/one/two/three/file.txt", Write}, + }, + }, + + {"add directory", + func(t *testing.T, tmp string) { + mkdirAll(t, tmp, "/one/two/three/four") + }, + func(t *testing.T, tmp string) { + mkdirAll(t, tmp, "one/two/new/dir") + touch(t, tmp, "one/two/new/file") + touch(t, tmp, "one/two/new/dir/file") + }, + Events{ + // TODO: don't see the new/dir being created; I guess this + // happens too fast; splitting out the mkdirAll() with + // eventSeparator in-between "fixes" it. May be resolved + // by #470. + {"/one/two/new", Create}, + {"/one/two/new/file", Create}, + {"/one/two/new/dir/file", Create}, + }, + }, + + // TODO: this test is flaky due to #470 + // {"remove directory", + // func(t *testing.T, tmp string) { + // mkdirAll(t, tmp, "/one/two/three/four") + // }, + // func(t *testing.T, tmp string) { + // cat(t, "asd", tmp, "one/two/three/file.txt") + // rmAll(t, tmp, "one/two") + // }, + // Events{ + // // TODO: this includes many duplicate events as we get a + // // notification both for the watch on the directory itself + // // as well as the parent that watches the directory. + // {"/one/two/three/file.txt", Create}, + // {"/one/two/three/file.txt", Remove}, + // {"/one/two/three/four", Remove}, + // {"/one/two/three/four", Remove}, + // {"/one/two/three", Remove}, + // {"/one/two/three", Remove}, + // {"/one/two", Remove}, + // {"/one/two", Remove}, + // }, + // }, + + { + "rename directory", + func(t *testing.T, tmp string) { + mkdirAll(t, tmp, "/one/two/three/four") + }, + func(t *testing.T, tmp string) { + mv(t, filepath.Join(tmp, "one"), tmp, "one-rename") + touch(t, tmp, "one-rename/file") + touch(t, tmp, "one-rename/two/three/file") + }, + Events{ + // TODO: rename + create + rename doesn't seem quite right. + {"/one", Rename}, + {"/one-rename", Create}, + {"/one-rename", Rename}, + {"/one-rename/file", Create}, + {"/one-rename/two/three/file", Create}, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tmp := t.TempDir() + w := newCollector(t) + + tt.preWatch(t, tmp) + addWatch(t, w.w, tmp, "...") + tt.postWatch(t, tmp) + + w.collect(t) + have := w.stop(t) + for i := range have { + have[i].Name = strings.TrimPrefix(have[i].Name, tmp) + } + + if have.String() != tt.want.String() { + t.Errorf("\nhave:\n%s\nwant:\n%s", have, tt.want) + } + }) + } +} diff --git a/kqueue.go b/kqueue.go index 6b8b3389..e9fec491 100644 --- a/kqueue.go +++ b/kqueue.go @@ -95,6 +95,11 @@ func (w *Watcher) Close() error { // Add starts watching the named file or directory (non-recursively). func (w *Watcher) Add(name string) error { + _, recurse := recursivePath(name) + if recurse { + return ErrRecursionUnsupported + } + w.mu.Lock() w.externalWatches[name] = true w.mu.Unlock() diff --git a/windows.go b/windows.go index 3140b8bb..a9c24ed4 100644 --- a/windows.go +++ b/windows.go @@ -81,6 +81,11 @@ func (w *Watcher) Add(name string) error { } w.mu.Unlock() + _, recurse := recursivePath(name) + if recurse { + return ErrRecursionUnsupported + } + in := &input{ op: opAddWatch, path: filepath.Clean(name),