store: retry RemoveAll on EBUSY

when running on NFS, a RemoveAll could cause EBUSY because of some
unlinked files that are still kept open and "silly renamed" to
.nfs$ID.

These files could be kept open by conmon and the issue is addressed
by: containers/conmon#300

Before failing, attempt a few more times.

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

pkg/system/syscall_unix.go

@@ -1,8 +1,11 @@
-// +build linux freebsd
+// +build linux freebsd darwin
 
 package system
 
-import "golang.org/x/sys/unix"
+import (
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+)
 
 // Unmount is a platform-specific helper function to call
 // the unmount syscall.
@@ -15,3 +18,8 @@ func Unmount(dest string) error {
 func CommandLineToArgv(commandLine string) ([]string, error) {
 	return []string{commandLine}, nil
 }
+
+// IsEBUSY checks if the specified error is EBUSY.
+func IsEBUSY(err error) bool {
+	return errors.Is(err, unix.EBUSY)
+}

pkg/system/syscall_windows.go

@@ -120,3 +120,8 @@ func HasWin32KSupport() bool {
 	// APIs.
 	return ntuserApiset.Load() == nil
 }
+
+// IsEBUSY checks if the specified error is EBUSY.
+func IsEBUSY(err error) bool {
+	return false
+}

store.go

@@ -23,6 +23,7 @@ import (
 	"github.com/containers/storage/pkg/parsers"
 	"github.com/containers/storage/pkg/stringid"
 	"github.com/containers/storage/pkg/stringutils"
+	"github.com/containers/storage/pkg/system"
 	"github.com/containers/storage/types"
 	"github.com/hashicorp/go-multierror"
 	digest "github.com/opencontainers/go-digest"
@@ -2498,7 +2499,15 @@ func (s *store) DeleteContainer(id string) error {
 			gcpath := filepath.Join(s.GraphRoot(), middleDir, container.ID)
 			wg.Add(1)
 			go func() {
-				errChan <- os.RemoveAll(gcpath)
+				var err error
+				for attempts := 0; attempts < 50; attempts++ {
+					err = os.RemoveAll(gcpath)
+					if err == nil || !system.IsEBUSY(err) {
+						break
+					}
+					time.Sleep(time.Millisecond * 100)
+				}
+				errChan <- err
 				wg.Done()
 			}()