diff --git a/CHANGELOG.md b/CHANGELOG.md
index ae493417c2f..aa8da90cecc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Correct `wrap_pymodule` to match normal namespacing rules: it no longer "sees through" glob imports of `use submodule::*` when `submodule::submodule` is a `#[pymodule]`. [#2363](https://github.com/PyO3/pyo3/pull/2363)
 - Allow `#[classattr]` methods to be fallible. [#2385](https://github.com/PyO3/pyo3/pull/2385)
 - Prevent multiple `#[pymethods]` with the same name for a single `#[pyclass]`. [#2399](https://github.com/PyO3/pyo3/pull/2399)
+- Fixup `lib_name` when using `PYO3_CONFIG_FILE`. [#2404](https://github.com/PyO3/pyo3/pull/2404)
 
 ### Fixed
 
diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs
index 4fa2a49cb1e..4fba11e2172 100644
--- a/pyo3-build-config/src/impl_.rs
+++ b/pyo3-build-config/src/impl_.rs
@@ -440,6 +440,14 @@ print("mingw", get_platform().startswith("mingw"))
         let version = version.ok_or("missing value for version")?;
         let implementation = implementation.unwrap_or(PythonImplementation::CPython);
         let abi3 = abi3.unwrap_or(false);
+        // Fixup lib_name if it's not set
+        let lib_name = lib_name.or_else(|| {
+            if let Ok(Ok(target)) = env::var("TARGET").map(|target| target.parse::<Triple>()) {
+                default_lib_name_for_target(version, implementation, abi3, &target)
+            } else {
+                None
+            }
+        });
 
         Ok(InterpreterConfig {
             implementation,
@@ -456,28 +464,24 @@ print("mingw", get_platform().startswith("mingw"))
         })
     }
 
+    #[cfg(feature = "python3-dll-a")]
     #[allow(clippy::unnecessary_wraps)]
-    pub fn fixup_import_libs(&mut self) -> Result<()> {
-        let target = target_triple_from_env();
-        if self.lib_name.is_none() && target.operating_system == OperatingSystem::Windows {
-            self.lib_name = Some(default_lib_name_windows(
-                self.version,
-                self.implementation,
-                self.abi3,
-                false,
-            ));
-        }
+    pub fn generate_import_libs(&mut self) -> Result<()> {
         // Auto generate python3.dll import libraries for Windows targets.
-        #[cfg(feature = "python3-dll-a")]
-        {
-            if self.lib_dir.is_none() {
-                let py_version = if self.abi3 { None } else { Some(self.version) };
-                self.lib_dir = self::import_lib::generate_import_lib(&target, py_version)?;
-            }
+        if self.lib_dir.is_none() {
+            let target = target_triple_from_env();
+            let py_version = if self.abi3 { None } else { Some(self.version) };
+            self.lib_dir = import_lib::generate_import_lib(&target, py_version)?;
         }
         Ok(())
     }
 
+    #[cfg(not(feature = "python3-dll-a"))]
+    #[allow(clippy::unnecessary_wraps)]
+    pub fn generate_import_libs(&mut self) -> Result<()> {
+        Ok(())
+    }
+
     #[doc(hidden)]
     /// Serialize the `InterpreterConfig` and print it to the environment for Cargo to pass along
     /// to dependent packages during build time.
@@ -1412,18 +1416,8 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<In
         .implementation
         .unwrap_or(PythonImplementation::CPython);
 
-    let lib_name = if cross_compile_config.target.operating_system == OperatingSystem::Windows {
-        Some(default_lib_name_windows(
-            version,
-            implementation,
-            abi3,
-            false,
-        ))
-    } else if is_linking_libpython_for_target(&cross_compile_config.target) {
-        Some(default_lib_name_unix(version, implementation, None))
-    } else {
-        None
-    };
+    let lib_name =
+        default_lib_name_for_target(version, implementation, abi3, &cross_compile_config.target);
 
     let mut lib_dir = cross_compile_config.lib_dir_string();
 
@@ -1532,6 +1526,26 @@ fn load_cross_compile_config(
 // This contains only the limited ABI symbols.
 const WINDOWS_ABI3_LIB_NAME: &str = "python3";
 
+fn default_lib_name_for_target(
+    version: PythonVersion,
+    implementation: PythonImplementation,
+    abi3: bool,
+    target: &Triple,
+) -> Option<String> {
+    if target.operating_system == OperatingSystem::Windows {
+        Some(default_lib_name_windows(
+            version,
+            implementation,
+            abi3,
+            false,
+        ))
+    } else if is_linking_libpython_for_target(target) {
+        Some(default_lib_name_unix(version, implementation, None))
+    } else {
+        None
+    }
+}
+
 fn default_lib_name_windows(
     version: PythonVersion,
     implementation: PythonImplementation,
diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs
index 6d20fd8f362..260ef0da745 100644
--- a/pyo3-build-config/src/lib.rs
+++ b/pyo3-build-config/src/lib.rs
@@ -178,7 +178,7 @@ pub mod pyo3_build_script_impl {
     pub fn resolve_interpreter_config() -> Result<InterpreterConfig> {
         if !CONFIG_FILE.is_empty() {
             let mut interperter_config = InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE))?;
-            interperter_config.fixup_import_libs()?;
+            interperter_config.generate_import_libs()?;
             Ok(interperter_config)
         } else if let Some(interpreter_config) = make_cross_compile_config()? {
             // This is a cross compile and need to write the config file.