diff --git a/packages/next/build/webpack-config.ts b/packages/next/build/webpack-config.ts
index cf1a86226c30938..1f45f69fa69672c 100644
--- a/packages/next/build/webpack-config.ts
+++ b/packages/next/build/webpack-config.ts
@@ -628,52 +628,59 @@ export default async function getBaseWebpackConfig(
     loggedIgnoredCompilerOptions = true
   }
 
-  const getBabelOrSwcLoader = () => {
-    if (useSWCLoader && config?.experimental?.swcTraceProfiling) {
+  const getBabelLoader = () => {
+    return {
+      loader: require.resolve('./babel/loader/index'),
+      options: {
+        configFile: babelConfigFile,
+        isServer: isNodeServer || isEdgeServer,
+        distDir,
+        pagesDir,
+        cwd: dir,
+        development: dev,
+        hasReactRefresh: dev && isClient,
+        hasJsxRuntime: true,
+      },
+    }
+  }
+
+  let swcTraceProfilingInitialized = false
+  const getSwcLoader = (extraOptions?: any) => {
+    if (
+      config?.experimental?.swcTraceProfiling &&
+      !swcTraceProfilingInitialized
+    ) {
       // This will init subscribers once only in a single process lifecycle,
       // even though it can be called multiple times.
       // Subscriber need to be initialized _before_ any actual swc's call (transform, etcs)
       // to collect correct trace spans when they are called.
+      swcTraceProfilingInitialized = true
       require('./swc')?.initCustomTraceSubscriber?.(
         path.join(distDir, `swc-trace-profile-${Date.now()}.json`)
       )
     }
 
-    return useSWCLoader
-      ? {
-          loader: 'next-swc-loader',
-          options: {
-            isServer: isNodeServer || isEdgeServer,
-            rootDir: dir,
-            pagesDir,
-            hasReactRefresh: dev && isClient,
-            fileReading: config.experimental.swcFileReading,
-            nextConfig: config,
-            jsConfig,
-            supportedBrowsers: config.experimental.browsersListForSwc
-              ? supportedBrowsers
-              : undefined,
-            swcCacheDir: path.join(
-              dir,
-              config?.distDir ?? '.next',
-              'cache',
-              'swc'
-            ),
-          },
-        }
-      : {
-          loader: require.resolve('./babel/loader/index'),
-          options: {
-            configFile: babelConfigFile,
-            isServer: isNodeServer || isEdgeServer,
-            distDir,
-            pagesDir,
-            cwd: dir,
-            development: dev,
-            hasReactRefresh: dev && isClient,
-            hasJsxRuntime: true,
-          },
-        }
+    return {
+      loader: 'next-swc-loader',
+      options: {
+        isServer: isNodeServer || isEdgeServer,
+        rootDir: dir,
+        pagesDir,
+        hasReactRefresh: dev && isClient,
+        fileReading: config.experimental.swcFileReading,
+        nextConfig: config,
+        jsConfig,
+        supportedBrowsers: config.experimental.browsersListForSwc
+          ? supportedBrowsers
+          : undefined,
+        swcCacheDir: path.join(dir, config?.distDir ?? '.next', 'cache', 'swc'),
+        ...extraOptions,
+      },
+    }
+  }
+
+  const getBabelOrSwcLoader = () => {
+    return useSWCLoader ? getSwcLoader() : getBabelLoader()
   }
 
   const defaultLoaders = {
@@ -1613,13 +1620,16 @@ export default async function getBaseWebpackConfig(
                   {
                     test: codeCondition.test,
                     issuerLayer: WEBPACK_LAYERS.server,
-                    use: {
-                      ...defaultLoaders.babel,
-                      options: {
-                        ...defaultLoaders.babel.options,
-                        isServerLayer: true,
-                      },
-                    },
+                    use: useSWCLoader
+                      ? getSwcLoader({ isServerLayer: true })
+                      : // When using Babel, we will have to add the SWC loader
+                        // as an additional pass to handle RSC correctly.
+                        // This will cause some performance overhead but
+                        // acceptable as Babel will not be recommended.
+                        [
+                          getSwcLoader({ isServerLayer: true }),
+                          getBabelLoader(),
+                        ],
                   },
                 ]
               : []),
diff --git a/test/e2e/app-dir/with-babel.test.ts b/test/e2e/app-dir/with-babel.test.ts
new file mode 100644
index 000000000000000..6244e00c7ed62a6
--- /dev/null
+++ b/test/e2e/app-dir/with-babel.test.ts
@@ -0,0 +1,35 @@
+import { createNext, FileRef } from 'e2e-utils'
+import { NextInstance } from 'test/lib/next-modes/base'
+import { renderViaHTTP } from 'next-test-utils'
+import path from 'path'
+import cheerio from 'cheerio'
+
+describe('with babel', () => {
+  if ((global as any).isNextDeploy) {
+    it('should skip next deploy for now', () => {})
+    return
+  }
+
+  if (process.env.NEXT_TEST_REACT_VERSION === '^17') {
+    it('should skip for react v17', () => {})
+    return
+  }
+  let next: NextInstance
+
+  beforeAll(async () => {
+    next = await createNext({
+      files: new FileRef(path.join(__dirname, 'with-babel')),
+      dependencies: {
+        react: 'experimental',
+        'react-dom': 'experimental',
+      },
+    })
+  })
+  afterAll(() => next.destroy())
+
+  it('should support babel in app dir', async () => {
+    const html = await renderViaHTTP(next.url, '/')
+    const $ = cheerio.load(html)
+    expect($('h1').text()).toBe('hello')
+  })
+})
diff --git a/test/e2e/app-dir/with-babel/app/layout.js b/test/e2e/app-dir/with-babel/app/layout.js
new file mode 100644
index 000000000000000..e66000e445cfc3d
--- /dev/null
+++ b/test/e2e/app-dir/with-babel/app/layout.js
@@ -0,0 +1,10 @@
+export default function Root({ children }) {
+  return (
+    <html>
+      <head>
+        <title>hello</title>
+      </head>
+      <body>{children}</body>
+    </html>
+  )
+}
diff --git a/test/e2e/app-dir/with-babel/app/page.js b/test/e2e/app-dir/with-babel/app/page.js
new file mode 100644
index 000000000000000..df85d684c2f35a8
--- /dev/null
+++ b/test/e2e/app-dir/with-babel/app/page.js
@@ -0,0 +1,8 @@
+'client'
+
+import { useState } from 'react'
+
+export default function Page() {
+  const state = useState('hello')[0]
+  return <h1>{state}</h1>
+}
diff --git a/test/e2e/app-dir/with-babel/babel.config.js b/test/e2e/app-dir/with-babel/babel.config.js
new file mode 100644
index 000000000000000..bc6c5100868f51f
--- /dev/null
+++ b/test/e2e/app-dir/with-babel/babel.config.js
@@ -0,0 +1,3 @@
+module.exports = {
+  presets: ['next/babel'],
+}
diff --git a/test/e2e/app-dir/with-babel/next.config.js b/test/e2e/app-dir/with-babel/next.config.js
new file mode 100644
index 000000000000000..cfa3ac3d7aa94b3
--- /dev/null
+++ b/test/e2e/app-dir/with-babel/next.config.js
@@ -0,0 +1,5 @@
+module.exports = {
+  experimental: {
+    appDir: true,
+  },
+}