Fixes for native fs-events

src/watchdog/observers/fsevents.py

@@ -111,7 +111,7 @@ def queue_events(self, timeout):
                         self.queue_event(DirModifiedEvent(os.path.dirname(event.path)))
                     # TODO: generate events for tree
 
-                elif event.is_modified or event.is_inode_meta_mod or event.is_xattr_mod :
+                elif event.is_modified or event.is_inode_meta_mod or event.is_xattr_mod:
                     cls = DirModifiedEvent if event.is_directory else FileModifiedEvent
                     self.queue_event(cls(event.path))
 

src/watchdog_fsevents.c

@@ -149,11 +149,29 @@ PyObject* NativeEventTypeID(PyObject* instance, void* closure)
         Py_RETURN_FALSE; \
     }
 
+FLAG_PROPERTY(IsMustScanSubDirs, kFSEventStreamEventFlagMustScanSubDirs)
+FLAG_PROPERTY(IsUserDropped, kFSEventStreamEventFlagUserDropped)
+FLAG_PROPERTY(IsKernelDropped, kFSEventStreamEventFlagKernelDropped)
+FLAG_PROPERTY(IsEventIdsWrapped, kFSEventStreamEventFlagEventIdsWrapped)
+FLAG_PROPERTY(IsHistoryDone, kFSEventStreamEventFlagHistoryDone)
+FLAG_PROPERTY(IsRootChanged, kFSEventStreamEventFlagRootChanged)
+FLAG_PROPERTY(IsMount, kFSEventStreamEventFlagMount)
+FLAG_PROPERTY(IsUnmount, kFSEventStreamEventFlagUnmount)
 FLAG_PROPERTY(IsCreated, kFSEventStreamEventFlagItemCreated)
 FLAG_PROPERTY(IsRemoved, kFSEventStreamEventFlagItemRemoved)
+FLAG_PROPERTY(IsInodeMetaMod, kFSEventStreamEventFlagItemInodeMetaMod)
 FLAG_PROPERTY(IsRenamed, kFSEventStreamEventFlagItemRenamed)
 FLAG_PROPERTY(IsModified, kFSEventStreamEventFlagItemModified)
+FLAG_PROPERTY(IsItemFinderInfoMod, kFSEventStreamEventFlagItemFinderInfoMod)
+FLAG_PROPERTY(IsChangeOwner, kFSEventStreamEventFlagItemChangeOwner)
+FLAG_PROPERTY(IsXattrMod, kFSEventStreamEventFlagItemXattrMod)
+FLAG_PROPERTY(IsFile, kFSEventStreamEventFlagItemIsFile)
 FLAG_PROPERTY(IsDirectory, kFSEventStreamEventFlagItemIsDir)
+FLAG_PROPERTY(IsSymlink, kFSEventStreamEventFlagItemIsSymlink)
+FLAG_PROPERTY(IsOwnEvent, kFSEventStreamEventFlagOwnEvent)
+FLAG_PROPERTY(IsHardlink, kFSEventStreamEventFlagItemIsHardlink)
+FLAG_PROPERTY(IsLastHardlink, kFSEventStreamEventFlagItemIsLastHardlink)
+FLAG_PROPERTY(IsCloned, kFSEventStreamEventFlagItemCloned)
 
 static int NativeEventInit(NativeEventObject *self, PyObject *args, PyObject *kwds)
 {
@@ -171,11 +189,29 @@ static PyGetSetDef NativeEventProperties[] = {
     {"flags", NativeEventTypeFlags, NULL, "The raw mask of flags as returend by FSEvents", NULL},
     {"path", NativeEventTypePath, NULL, "The path for which this event was generated", NULL},
     {"id", NativeEventTypeID, NULL, "The id of the generated event", NULL},
+    {"must_scan_subdirs", NativeEventTypeIsMustScanSubDirs, NULL, "True if application must rescan all subdirectories", NULL},
+    {"is_user_dropped", NativeEventTypeIsUserDropped, NULL, "True if a failure during event buffering occured", NULL},
+    {"is_kernel_dropped", NativeEventTypeIsKernelDropped, NULL, "True if a failure during event buffering occured", NULL},
+    {"is_event_ids_wrapped", NativeEventTypeIsEventIdsWrapped, NULL, "True if event_id wrapped around", NULL},
+    {"is_history_done", NativeEventTypeIsHistoryDone, NULL, "True if all historical events are done", NULL},
+    {"is_root_changed", NativeEventTypeIsRootChanged, NULL, "True if a change to one of the directories along the path to one of the directories you watch occurred", NULL},
+    {"is_mount", NativeEventTypeIsMount, NULL, "True if a volume is mounted underneath one of the paths being monitored", NULL},
+    {"is_unmount", NativeEventTypeIsUnmount, NULL, "True if a volume is unmounted underneath one of the paths being monitored", NULL},
     {"is_created", NativeEventTypeIsCreated, NULL, "True if self.path was created on the filesystem", NULL},
     {"is_removed", NativeEventTypeIsRemoved, NULL, "True if self.path was removed from the filesystem", NULL},
+    {"is_inode_meta_mod", NativeEventTypeIsInodeMetaMod, NULL, "True if meta data for self.path was modified ", NULL},
     {"is_renamed", NativeEventTypeIsRenamed, NULL, "True if self.path was renamed on the filesystem", NULL},
     {"is_modified", NativeEventTypeIsModified, NULL, "True if self.path was modified", NULL},
+    {"is_item_finder_info_modified", NativeEventTypeIsItemFinderInfoMod, NULL, "True if FinderInfo for self.path was modified", NULL},
+    {"is_owner_change", NativeEventTypeIsChangeOwner, NULL, "True if self.path had its ownership changed", NULL},
+    {"is_xattr_mod", NativeEventTypeIsXattrMod, NULL, "True if extended attributes for self.path were modified ", NULL},
+    {"is_file", NativeEventTypeIsFile, NULL, "True if self.path is a file", NULL},
     {"is_directory", NativeEventTypeIsDirectory, NULL, "True if self.path is a directory", NULL},
+    {"is_symlink", NativeEventTypeIsSymlink, NULL, "True if self.path is a symbolic link", NULL},
+    {"is_own_event", NativeEventTypeIsOwnEvent, NULL, "True if the event originated from our own process", NULL},
+    {"is_hardlink", NativeEventTypeIsHardlink, NULL, "True if self.path is a hard link", NULL},
+    {"is_last_hardlink", NativeEventTypeIsLastHardlink, NULL, "True if self.path was the last hard link", NULL},
+    {"is_cloned", NativeEventTypeIsCloned, NULL, "True if self.path is a clone or was cloned", NULL},
     {NULL, NULL, NULL, NULL, NULL},
 };
 
@@ -207,17 +243,15 @@ PyObject *watch_to_stream = NULL;
 
 
 /**
- * PyCapsule destructor for Python 3 compatibility
+ * PyCapsule destructor
  */
-#if PY_MAJOR_VERSION >= 3
 static void watchdog_pycapsule_destructor(PyObject *ptr)
 {
     void *p = PyCapsule_GetPointer(ptr, NULL);
     if (p) {
         PyMem_Free(p);
     }
 }
-#endif
 
 
 /**
@@ -272,14 +306,14 @@ watchdog_FSEventStreamCallback(ConstFSEventStreamRef          stream_ref,
     py_event_ids = PyList_New(num_events);
     if (G_NOT(py_event_paths && py_event_flags && py_event_ids))
     {
-        Py_DECREF(py_event_paths);
-        Py_DECREF(py_event_ids);
-        Py_DECREF(py_event_flags);
+        Py_XDECREF(py_event_paths);
+        Py_XDECREF(py_event_ids);
+        Py_XDECREF(py_event_flags);
         return /*NULL*/;
     }
     for (i = 0; i < num_events; ++i)
     {
-        id = PyLong_FromLongLong(event_flags[i]);
+        id = PyLong_FromLongLong(event_ids[i]);
 #if PY_MAJOR_VERSION >= 3
         path = PyUnicode_FromString(event_paths[i]);
         flags = PyLong_FromLong(event_flags[i]);
@@ -323,6 +357,40 @@ watchdog_FSEventStreamCallback(ConstFSEventStreamRef          stream_ref,
     PyGILState_Release(gil_state);
 }
 
+/**
+ * Converts a Python string object to an UTF-8 encoded ``CFStringRef``.
+ *
+ * :param py_string:
+ *      A Python unicode or utf-8 encoded bytestring object.
+ * :returns:
+ *      A new ``CFStringRef`` with the contents of ``py_string``, or ``NULL`` if an error occurred.
+ */
+CFStringRef PyString_AsUTF8EncodedCFStringRef(PyObject *py_string)
+{
+    CFStringRef cf_string = NULL;
+
+    if (PyUnicode_Check(py_string)) {
+        PyObject* helper = PyUnicode_AsUTF8String(py_string);
+        if (!helper) {
+            return NULL;
+        }
+        cf_string = CFStringCreateWithCString(kCFAllocatorDefault, PyBytes_AS_STRING(helper), kCFStringEncodingUTF8);
+        Py_DECREF(helper);
+    } else if (PyBytes_Check(py_string)) {
+        PyObject *utf8 = PyUnicode_FromEncodedObject(py_string, NULL, "strict");
+        if (!utf8) {
+            return NULL;
+        }
+        Py_DECREF(utf8);
+        cf_string = CFStringCreateWithCString(kCFAllocatorDefault, PyBytes_AS_STRING(py_string), kCFStringEncodingUTF8);
+    } else {
+        PyErr_SetString(PyExc_TypeError, "Path to watch must be a string or a UTF-8 encoded bytes object.");
+        return NULL;
+    }
+
+    return cf_string;
+}
+
 
 /**
  * Converts a list of Python strings to a ``CFMutableArray`` of
@@ -341,7 +409,6 @@ watchdog_CFMutableArrayRef_from_PyStringList(PyObject *py_string_list)
 {
     Py_ssize_t i = 0;
     Py_ssize_t string_list_size = 0;
-    const char *c_string = NULL;
     CFMutableArrayRef array_of_cf_string = NULL;
     CFStringRef cf_string = NULL;
     PyObject *py_string = NULL;
@@ -361,18 +428,10 @@ watchdog_CFMutableArrayRef_from_PyStringList(PyObject *py_string_list)
     {
         py_string = PyList_GetItem(py_string_list, i);
         G_RETURN_NULL_IF_NULL(py_string);
-#if PY_MAJOR_VERSION >= 3
-        if (PyUnicode_Check(py_string)) {
-          c_string = PyUnicode_AsUTF8(py_string);
-        } else {
-          c_string = PyBytes_AS_STRING(py_string);
-        }
-#else
-        c_string = PyString_AS_STRING(py_string);
-#endif
-        cf_string = CFStringCreateWithCString(kCFAllocatorDefault,
-                                              c_string,
-                                              kCFStringEncodingUTF8);
+
+        cf_string = PyString_AsUTF8EncodedCFStringRef(py_string);
+        G_RETURN_NULL_IF_NULL(cf_string);
+
         CFArraySetValueAtIndex(array_of_cf_string, i, cf_string);
         CFRelease(cf_string);
     }
@@ -479,11 +538,18 @@ watchdog_add_watch(PyObject *self, PyObject *args)
     stream_ref = watchdog_FSEventStreamCreate(stream_callback_info_ref,
                                               paths_to_watch,
                                               (FSEventStreamCallback) &watchdog_FSEventStreamCallback);
-#if PY_MAJOR_VERSION >= 3
+    if (!stream_ref) {
+        PyMem_Del(stream_callback_info_ref);
+        PyErr_SetString(PyExc_RuntimeError, "Failed creating fsevent stream");
+        return NULL;
+    }
     value = PyCapsule_New(stream_ref, NULL, watchdog_pycapsule_destructor);
-#else
-    value = PyCObject_FromVoidPtr(stream_ref, PyMem_Free);
-#endif
+    if (!value || !PyCapsule_IsValid(value, NULL)) {
+        PyMem_Del(stream_callback_info_ref);
+        FSEventStreamInvalidate(stream_ref);
+        FSEventStreamRelease(stream_ref);
+        return NULL;
+    }
     PyDict_SetItem(watch_to_stream, watch, value);
 
     /* Get a reference to the runloop for the emitter thread
@@ -495,11 +561,7 @@ watchdog_add_watch(PyObject *self, PyObject *args)
     }
     else
     {
-#if PY_MAJOR_VERSION >= 3
         run_loop_ref = PyCapsule_GetPointer(value, NULL);
-#else
-        run_loop_ref = PyCObject_AsVoidPtr(value);
-#endif
     }
 
     /* Schedule the stream with the obtained runloop. */
@@ -550,11 +612,7 @@ watchdog_read_events(PyObject *self, PyObject *args)
     if (G_IS_NULL(value))
     {
         run_loop_ref = CFRunLoopGetCurrent();
-#if PY_MAJOR_VERSION >= 3
         value = PyCapsule_New(run_loop_ref, NULL, watchdog_pycapsule_destructor);
-#else
-        value = PyCObject_FromVoidPtr(run_loop_ref, PyMem_Free);
-#endif
         PyDict_SetItem(thread_to_run_loop, emitter_thread, value);
         Py_INCREF(emitter_thread);
         Py_INCREF(value);
@@ -590,11 +648,7 @@ watchdog_remove_watch(PyObject *self, PyObject *watch)
     PyObject *value = PyDict_GetItem(watch_to_stream, watch);
     PyDict_DelItem(watch_to_stream, watch);
 
-#if PY_MAJOR_VERSION >= 3
     FSEventStreamRef stream_ref = PyCapsule_GetPointer(value, NULL);
-#else
-    FSEventStreamRef stream_ref = PyCObject_AsVoidPtr(value);
-#endif
 
     FSEventStreamStop(stream_ref);
     FSEventStreamInvalidate(stream_ref);
@@ -618,11 +672,7 @@ watchdog_stop(PyObject *self, PyObject *emitter_thread)
       goto success;
     }
 
-#if PY_MAJOR_VERSION >= 3
     CFRunLoopRef run_loop_ref = PyCapsule_GetPointer(value, NULL);
-#else
-    CFRunLoopRef run_loop_ref = PyCObject_AsVoidPtr(value);
-#endif
     G_RETURN_NULL_IF(PyErr_Occurred());
 
     /* Stop the run loop. */

tests/test_fsevents.py

@@ -12,12 +12,13 @@
 from functools import partial
 from os import mkdir, rmdir
 
+from watchdog.events import FileSystemEventHandler
 from watchdog.observers import Observer
 from watchdog.observers.api import ObservedWatch
 from watchdog.observers.fsevents import FSEventsEmitter
 
 from . import Queue
-from .shell import mkdtemp, rm
+from .shell import mkdtemp, rm, touch
 
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
@@ -96,7 +97,49 @@ def on_thread_stop(self):
     """
     a = p("a")
     mkdir(a)
-    w = observer.schedule(event_queue, a, recursive=False)
+    w = observer.schedule(FileSystemEventHandler(), a, recursive=False)
     rmdir(a)
     time.sleep(0.1)
     observer.unschedule(w)
+
+
+def test_watchdog_recursive():
+    """ See https://github.com/gorakhargosh/watchdog/issues/706
+    """
+    from watchdog.observers import Observer
+    from watchdog.events import FileSystemEventHandler
+    import os.path
+
+    class Handler(FileSystemEventHandler):
+        def __init__(self):
+            FileSystemEventHandler.__init__(self)
+            self.changes = []
+
+        def on_any_event(self, event):
+            self.changes.append(os.path.basename(event.src_path))
+
+    handler = Handler()
+    observer = Observer()
+
+    watches = []
+    watches.append(observer.schedule(handler, str(p('')), recursive=True))
+
+    try:
+        observer.start()
+        time.sleep(0.1)
+
+        touch(p('my0.txt'))
+        mkdir(p('dir_rec'))
+        touch(p('dir_rec', 'my1.txt'))
+
+        expected = {"dir_rec", "my0.txt", "my1.txt"}
+        timeout_at = time.time() + 5
+        while not expected.issubset(handler.changes) and time.time() < timeout_at:
+            time.sleep(0.2)
+
+        assert expected.issubset(handler.changes), "Did not find expected changes. Found: {}".format(handler.changes)
+    finally:
+        for watch in watches:
+            observer.unschedule(watch)
+        observer.stop()
+        observer.join(1)

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py{310,39,38,37,36,35,27,py3,py}
+envlist = py{35,27,py}
 skip_missing_interpreters = True
 
 [testenv]