diff --git a/packages/next/build/babel/loader/get-config.ts b/packages/next/build/babel/loader/get-config.ts
index 5c4855d055e789c..c0d80a805b468f2 100644
--- a/packages/next/build/babel/loader/get-config.ts
+++ b/packages/next/build/babel/loader/get-config.ts
@@ -302,6 +302,7 @@ function getCacheKey(cacheCharacteristics: CharacteristicsGermaneToCaching) {
 
 type BabelConfig = any
 const configCache: Map<any, BabelConfig> = new Map()
+const configFiles: Set<string> = new Set()
 
 export default function getConfig(
   this: NextJsLoaderContext,
@@ -346,7 +347,8 @@ export default function getConfig(
     }
   }
 
-  if (loaderOptions.configFile) {
+  if (loaderOptions.configFile && !configFiles.has(loaderOptions.configFile)) {
+    configFiles.add(loaderOptions.configFile)
     Log.info(
       `Using external babel configuration from ${loaderOptions.configFile}`
     )