diff --git a/drivers/driver_linux.go b/drivers/driver_linux.go
index dddf8a8b4c..0fe3eea7ae 100644
--- a/drivers/driver_linux.go
+++ b/drivers/driver_linux.go
@@ -50,6 +50,40 @@ const (
 	FsMagicOverlay = FsMagic(0x794C7630)
 	// FsMagicFUSE filesystem id for FUSE
 	FsMagicFUSE = FsMagic(0x65735546)
+	// FsMagicAcfs filesystem id for Acfs
+	FsMagicAcfs = FsMagic(0x61636673)
+	// FsMagicAfs filesystem id for Afs
+	FsMagicAfs = FsMagic(0x5346414f)
+	// FsMagicCephFs filesystem id for Ceph
+	FsMagicCephFs = FsMagic(0x00C36400)
+	// FsMagicCIFS filesystem id for CIFS
+	FsMagicCIFS = FsMagic(0xFF534D42)
+	// FsMagicFHGFS filesystem id for FHGFS
+	FsMagicFHGFSFs = FsMagic(0x19830326)
+	// FsMagicIBRIX filesystem id for IBRIX
+	FsMagicIBRIX = FsMagic(0x013111A8)
+	// FsMagicKAFS filesystem id for KAFS
+	FsMagicKAFS = FsMagic(0x6B414653)
+	// FsMagicLUSTRE filesystem id for LUSTRE
+	FsMagicLUSTRE = FsMagic(0x0BD00BD0)
+	// FsMagicNCP filesystem id for NCP
+	FsMagicNCP = FsMagic(0x564C)
+	// FsMagicNFSD filesystem id for NFSD
+	FsMagicNFSD = FsMagic(0x6E667364)
+	// FsMagicOCFS2 filesystem id for OCFS2
+	FsMagicOCFS2 = FsMagic(0x7461636F)
+	// FsMagicPANFS filesystem id for PANFS
+	FsMagicPANFS = FsMagic(0xAAD7AAEA)
+	// FsMagicPRLFS filesystem id for PRLFS
+	FsMagicPRLFS = FsMagic(0x7C7C6673)
+	// FsMagicSMB2 filesystem id for SMB2
+	FsMagicSMB2 = FsMagic(0xFE534D42)
+	// FsMagicSNFS filesystem id for SNFS
+	FsMagicSNFS = FsMagic(0xBEEFDEAD)
+	// FsMagicVBOXSF filesystem id for VBOXSF
+	FsMagicVBOXSF = FsMagic(0x786F4256)
+	// FsMagicVXFS filesystem id for VXFS
+	FsMagicVXFS = FsMagic(0xA501FCF5)
 )
 
 var (
diff --git a/drivers/overlay/overlay.go b/drivers/overlay/overlay.go
index 62130c73e7..1efe7316d3 100644
--- a/drivers/overlay/overlay.go
+++ b/drivers/overlay/overlay.go
@@ -248,6 +248,23 @@ func (d *Driver) getSupportsVolatile() (bool, error) {
 	return supportsVolatile, nil
 }
 
+// isNetworkFileSystem checks if the specified file system is supported by native overlay
+// as backing store when running in a user namespace.
+func isNetworkFileSystem(fsMagic graphdriver.FsMagic) bool {
+	switch fsMagic {
+	// a bunch of network file systems...
+	case graphdriver.FsMagicNfsFs, graphdriver.FsMagicSmbFs, graphdriver.FsMagicAcfs,
+		graphdriver.FsMagicAfs, graphdriver.FsMagicCephFs, graphdriver.FsMagicCIFS,
+		graphdriver.FsMagicFHGFSFs, graphdriver.FsMagicGPFS, graphdriver.FsMagicIBRIX,
+		graphdriver.FsMagicKAFS, graphdriver.FsMagicLUSTRE, graphdriver.FsMagicNCP,
+		graphdriver.FsMagicNFSD, graphdriver.FsMagicOCFS2, graphdriver.FsMagicPANFS,
+		graphdriver.FsMagicPRLFS, graphdriver.FsMagicSMB2, graphdriver.FsMagicSNFS,
+		graphdriver.FsMagicVBOXSF, graphdriver.FsMagicVXFS:
+		return true
+	}
+	return false
+}
+
 // Init returns the a native diff driver for overlay filesystem.
 // If overlay filesystem is not supported on the host, a wrapped graphdriver.ErrNotSupported is returned as error.
 // If an overlay filesystem is not supported over an existing filesystem then a wrapped graphdriver.ErrIncompatibleFS is returned.
@@ -266,18 +283,27 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error)
 	}
 
 	if opts.mountProgram != "" {
+		if unshare.IsRootless() && isNetworkFileSystem(fsMagic) && opts.forceMask == nil {
+			m := os.FileMode(0700)
+			opts.forceMask = &m
+			logrus.Warnf("Network file system detected as backing store.  Enforcing overlay option `force_mask=\"%o\"`.  Add it to storage.conf to silence this warning", m)
+		}
+
 		if err := ioutil.WriteFile(getMountProgramFlagFile(home), []byte("true"), 0600); err != nil {
 			return nil, err
 		}
 	} else {
-		// check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs
 		if opts.forceMask != nil {
 			return nil, errors.New("'force_mask' is supported only with 'mount_program'")
 		}
+		// check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs
 		switch fsMagic {
 		case graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs:
 			return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' is not supported over %s, a mount_program is required", backingFs)
 		}
+		if unshare.IsRootless() && isNetworkFileSystem(fsMagic) {
+			return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "A network file system with user namespaces is not supported.  Please use a mount_program")
+		}
 	}
 
 	rootUID, rootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps)
@@ -1431,6 +1457,11 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
 				label = d.optsAppendMappings(label, options.UidMaps, options.GidMaps)
 			}
 
+			// if forceMask is in place, tell fuse-overlayfs to write the permissions mask to an unprivileged xattr as well.
+			if d.options.forceMask != nil {
+				label = label + ",xattr_permissions=2"
+			}
+
 			mountProgram := exec.Command(d.options.mountProgram, "-o", label, target)
 			mountProgram.Dir = d.home
 			var b bytes.Buffer
diff --git a/pkg/system/syscall_unix.go b/pkg/system/syscall_unix.go
index 49dbdd3781..1bb852d11f 100644
--- a/pkg/system/syscall_unix.go
+++ b/pkg/system/syscall_unix.go
@@ -1,8 +1,11 @@
-// +build linux freebsd
+// +build linux freebsd darwin
 
 package system
 
-import "golang.org/x/sys/unix"
+import (
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
 
 // Unmount is a platform-specific helper function to call
 // the unmount syscall.
@@ -15,3 +18,8 @@ func Unmount(dest string) error {
 func CommandLineToArgv(commandLine string) ([]string, error) {
 	return []string{commandLine}, nil
 }
+
+// IsEBUSY checks if the specified error is EBUSY.
+func IsEBUSY(err error) bool {
+	return errors.Is(err, unix.EBUSY)
+}
diff --git a/pkg/system/syscall_windows.go b/pkg/system/syscall_windows.go
index 23e9b207c7..f4d8692cdb 100644
--- a/pkg/system/syscall_windows.go
+++ b/pkg/system/syscall_windows.go
@@ -120,3 +120,8 @@ func HasWin32KSupport() bool {
 	// APIs.
 	return ntuserApiset.Load() == nil
 }
+
+// IsEBUSY checks if the specified error is EBUSY.
+func IsEBUSY(err error) bool {
+	return false
+}
diff --git a/store.go b/store.go
index 6f6f698072..169c7d1513 100644
--- a/store.go
+++ b/store.go
@@ -23,6 +23,7 @@ import (
 	"github.com/containers/storage/pkg/parsers"
 	"github.com/containers/storage/pkg/stringid"
 	"github.com/containers/storage/pkg/stringutils"
+	"github.com/containers/storage/pkg/system"
 	"github.com/containers/storage/types"
 	"github.com/hashicorp/go-multierror"
 	digest "github.com/opencontainers/go-digest"
@@ -2498,7 +2499,15 @@ func (s *store) DeleteContainer(id string) error {
 			gcpath := filepath.Join(s.GraphRoot(), middleDir, container.ID)
 			wg.Add(1)
 			go func() {
-				errChan <- os.RemoveAll(gcpath)
+				var err error
+				for attempts := 0; attempts < 50; attempts++ {
+					err = os.RemoveAll(gcpath)
+					if err == nil || !system.IsEBUSY(err) {
+						break
+					}
+					time.Sleep(time.Millisecond * 100)
+				}
+				errChan <- err
 				wg.Done()
 			}()