diff --git a/go-selinux/rchcon.go b/go-selinux/rchcon.go
index fa1440d..8bff293 100644
--- a/go-selinux/rchcon.go
+++ b/go-selinux/rchcon.go
@@ -12,7 +12,18 @@ import (
 )
 
 func rchcon(fpath, label string) error {
+	fastMode := false
+	// If the current label matches the new label, assume
+	// other labels are correct.
+	if cLabel, err := lFileLabel(fpath); err == nil && cLabel == label {
+		fastMode = true
+	}
 	return pwalkdir.Walk(fpath, func(p string, _ fs.DirEntry, _ error) error {
+		if fastMode {
+			if cLabel, err := lFileLabel(fpath); err == nil && cLabel == label {
+				return nil
+			}
+		}
 		e := lSetFileLabel(p, label)
 		// Walk a file tree can race with removal, so ignore ENOENT.
 		if errors.Is(e, os.ErrNotExist) {
diff --git a/go-selinux/selinux_linux.go b/go-selinux/selinux_linux.go
index d56c317..4582cc9 100644
--- a/go-selinux/selinux_linux.go
+++ b/go-selinux/selinux_linux.go
@@ -1129,7 +1129,24 @@ func chcon(fpath string, label string, recurse bool) error {
 	}
 
 	if !recurse {
-		return setFileLabel(fpath, label)
+		err := lSetFileLabel(fpath, label)
+		if err != nil {
+			// Check if file doesn't exist, must have been removed
+			if errors.Is(err, os.ErrNotExist) {
+				return nil
+			}
+			// Check if current label is correct on disk
+			flabel, nerr := lFileLabel(fpath)
+			if nerr == nil && flabel == label {
+				return nil
+			}
+			// Check if file doesn't exist, must have been removed
+			if errors.Is(nerr, os.ErrNotExist) {
+				return nil
+			}
+			return err
+		}
+		return nil
 	}
 
 	return rchcon(fpath, label)