diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts
index de09f978cd505c..b31cd7a1b595bf 100644
--- a/packages/vite/src/node/optimizer/index.ts
+++ b/packages/vite/src/node/optimizer/index.ts
@@ -15,6 +15,7 @@ import {
   lookupFile,
   normalizeId,
   normalizePath,
+  removeDir,
   removeDirSync,
   renameDir,
   writeFile
@@ -534,7 +535,7 @@ export async function runOptimizeDeps(
   async function commitProcessingDepsCacheSync() {
     // Processing is done, we can now replace the depsCacheDir with processingCacheDir
     // Rewire the file paths from the temporal processing dir to the final deps cache dir
-    removeDirSync(depsCacheDir)
+    await removeDir(depsCacheDir)
     await renameDir(processingCacheDir, depsCacheDir)
   }
 
diff --git a/packages/vite/src/node/utils.ts b/packages/vite/src/node/utils.ts
index 70eb3fb8229b81..048193e09f6537 100644
--- a/packages/vite/src/node/utils.ts
+++ b/packages/vite/src/node/utils.ts
@@ -534,6 +534,9 @@ export function removeDirSync(dir: string) {
   }
 }
 
+export const removeDir = isWindows
+  ? promisify(gracefulRemoveDir)
+  : removeDirSync
 export const renameDir = isWindows ? promisify(gracefulRename) : fs.renameSync
 
 export function ensureWatchedFile(
@@ -815,6 +818,38 @@ function gracefulRename(
   })
 }
 
+const GRACEFUL_REMOVE_DIR_TIMEOUT = 5000
+function gracefulRemoveDir(
+  dir: string,
+  cb: (error: NodeJS.ErrnoException | null) => void
+) {
+  const rmdir = fs.rm ?? fs.rmdir // TODO: Remove after support for Node 12 is dropped
+  const start = Date.now()
+  let backoff = 0
+  rmdir(dir, { recursive: true }, function CB(er) {
+    if (er) {
+      if (
+        (er.code === 'ENOTEMPTY' ||
+          er.code === 'EACCES' ||
+          er.code === 'EPERM') &&
+        Date.now() - start < GRACEFUL_REMOVE_DIR_TIMEOUT
+      ) {
+        setTimeout(function () {
+          rmdir(dir, { recursive: true }, CB)
+        }, backoff)
+        if (backoff < 100) backoff += 10
+        return
+      }
+
+      if (er.code === 'ENOENT') {
+        er = null
+      }
+    }
+
+    if (cb) cb(er)
+  })
+}
+
 export function emptyCssComments(raw: string) {
   return raw.replace(multilineCommentsRE, (s) => ' '.repeat(s.length))
 }