Improve handling of fsnotify events for container rootfs removal.

Prior to this commit, sysbox-mgr only monitored fsnotify events
that signaled the removal of a container's rootfs.

However in some situations it appears fsnotify can miss the notification or
notify via a different event type (see fsnotify/fsnotify#404).

This commit causes sysbox-mgr to explicitly check for container rootfs removal
whenever it receives any fsnotify event.

In addition, update the fsnotify version from v1.4.7 -> v1.5.4 to leverage
several improvements and bug fixes.

Signed-off-by: Cesar Talledo <cesar.talledo@docker.com>

go.mod

@@ -5,7 +5,7 @@ go 1.13
 require (
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/deckarep/golang-set v1.7.1
-	github.com/fsnotify/fsnotify v1.4.7
+	github.com/fsnotify/fsnotify v1.5.4
 	github.com/google/uuid v1.1.2
 	github.com/nestybox/sysbox-ipc v0.0.0-00010101000000-000000000000
 	github.com/nestybox/sysbox-libs/dockerUtils v0.0.0-00010101000000-000000000000
@@ -18,7 +18,7 @@ require (
 	github.com/pkg/profile v1.5.0
 	github.com/sirupsen/logrus v1.7.0
 	github.com/urfave/cli v1.22.1
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad
 )
 
 replace github.com/nestybox/sysbox-ipc => ../sysbox-ipc

go.sum

@@ -39,8 +39,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
+github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -167,8 +167,9 @@ golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201029080932-201ba4db2418/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

mgr.go

@@ -647,7 +647,7 @@ func (mgr *SysboxMgr) unregister(id string) error {
 			delete(mgr.rootfsTable, origRootfs)
 			mgr.rootfsWatcher.Remove(origRootfs)
 			mgr.rtLock.Unlock()
-			mgr.removeCont(id)
+			go mgr.removeCont(id)
 			return nil
 		}
 
@@ -700,8 +700,11 @@ func (mgr *SysboxMgr) rootfsMon() {
 	for {
 		select {
 		case event := <-mgr.rootfsWatcher.Events:
-			if event.Op&fsnotify.Remove == fsnotify.Remove {
-				rootfs := event.Name
+			rootfs := event.Name
+			_, err := os.Stat(rootfs)
+
+			if event.Op&fsnotify.Remove == fsnotify.Remove || os.IsNotExist(err) {
+				logrus.Debugf("rootfsMon: rm event on %s", rootfs)
 				mgr.rtLock.Lock()
 				id, found := mgr.rootfsTable[rootfs]
 				if !found {
@@ -711,11 +714,11 @@ func (mgr *SysboxMgr) rootfsMon() {
 					mgr.rtLock.Unlock()
 					break
 				}
-				logrus.Debugf("rootfsMon: rm on %s", rootfs)
 				delete(mgr.rootfsTable, rootfs)
 				mgr.rtLock.Unlock()
 				mgr.rootfsWatcher.Remove(rootfs)
-				mgr.removeCont(id)
+
+				go mgr.removeCont(id)
 			}
 
 		case err := <-mgr.rootfsWatcher.Errors: