From ae7e1f5ee947a570d9e44e9dd97028d0f37ad64c Mon Sep 17 00:00:00 2001 From: Sergey Kvachonok Date: Mon, 11 Apr 2022 14:45:16 +0300 Subject: [PATCH] pyo3-build-config: Build "abi3" extensions without an interpreter Support compiling portable "abi3" extension modules even when the build host Python interpreter configuration is not available or the discovered Python interpreter version is not supported. Maturin already implements this by building "abi3" extension wheels with `PYO3_NO_PYTHON` environment veriable set for cargo when an `abi3-py3*` feature is detected. Closes #2292 --- CHANGELOG.md | 1 + Cargo.toml | 2 +- guide/src/building_and_distribution.md | 2 + pyo3-build-config/Cargo.toml | 4 ++ pyo3-build-config/src/impl_.rs | 77 ++++++++++++++++++++------ 5 files changed, 69 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c2b74a3695..96e8428c893 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Allow to compile "abi3" extensions without a working build host Python interpreter. [#2293](https://github.com/PyO3/pyo3/pull/2293) - Default to "m" ABI tag when choosing `libpython` link name for CPython 3.7 on Unix. [#2288](https://github.com/PyO3/pyo3/pull/2288) - Improved performance of failing calls to `FromPyObject::extract` which is common when functions accept multiple distinct types. [#2279](https://github.com/PyO3/pyo3/pull/2279) diff --git a/Cargo.toml b/Cargo.toml index 99655923824..833ac0aca52 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,7 +67,7 @@ pyproto = ["pyo3-macros/pyproto"] # Use this feature when building an extension module. # It tells the linker to keep the python symbols unresolved, # so that the module can also be used with statically linked python interpreters. -extension-module = ["pyo3-ffi/extension-module"] +extension-module = ["pyo3-build-config/extension-module", "pyo3-ffi/extension-module"] # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more. abi3 = ["pyo3-build-config/abi3", "pyo3-ffi/abi3", "pyo3-macros/abi3"] diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 0fda63b30fa..ce69030ca90 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -157,6 +157,8 @@ PyO3 is only able to link your extension module to api3 version up to and includ #### Building `abi3` extensions without a Python interpreter As an advanced feature, you can build PyO3 wheel without calling Python interpreter with the environment variable `PYO3_NO_PYTHON` set. +Also, if the build host Python interpreter is not found or is too old or otherwise unusable, +PyO3 will still attempt to compile `abi3` extension modules after displaying a warning message. On Unix-like systems this works unconditionally; on Windows you must also set the `RUSTFLAGS` environment variable to contain `-L native=/path/to/python/libs` so that the linker can find `python3.lib`. diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 86ffa36e101..7449b1f97be 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -26,6 +26,10 @@ default = [] # script. If this feature isn't enabled, the build script no-ops. resolve-config = [] +# This feature is enabled by pyo3 when building an extension module. +extension-module = [] + +# These features are enabled by pyo3 when building Stable ABI extension modules. abi3 = [] abi3-py37 = ["abi3-py38"] abi3-py38 = ["abi3-py39"] diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index d34cc7a1ac6..07bd510262c 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -233,6 +233,13 @@ print("mingw", get_platform().startswith("mingw")) "#; let output = run_python_script(interpreter.as_ref(), SCRIPT)?; let map: HashMap = parse_script_output(&output); + + ensure!( + !map.is_empty(), + "broken Python interpreter: {}", + interpreter.as_ref().display() + ); + let shared = map["shared"].as_str() == "True"; let version = PythonVersion { @@ -682,14 +689,14 @@ pub fn is_extension_module() -> bool { /// Checks if we need to link to `libpython` for the current build target. /// -/// Must be called from a crate PyO3 build script. +/// Must be called from a PyO3 crate build script. pub fn is_linking_libpython() -> bool { is_linking_libpython_for_target(&target_triple_from_env()) } /// Checks if we need to link to `libpython` for the target. /// -/// Must be called from a crate PyO3 build script. +/// Must be called from a PyO3 crate build script. fn is_linking_libpython_for_target(target: &Triple) -> bool { target.operating_system == OperatingSystem::Windows || target.environment == Environment::Android @@ -697,6 +704,18 @@ fn is_linking_libpython_for_target(target: &Triple) -> bool { || !is_extension_module() } +/// Checks if we need to discover the Python library directory +/// to link the extension module binary. +/// +/// Must be called from a PyO3 crate build script. +fn require_libdir_for_target(target: &Triple) -> bool { + let is_generating_libpython = cfg!(feature = "python3-dll-a") + && target.operating_system == OperatingSystem::Windows + && is_abi3(); + + is_linking_libpython_for_target(target) && !is_generating_libpython +} + /// Configuration needed by PyO3 to cross-compile for a target platform. /// /// Usually this is collected from the environment (i.e. `PYO3_CROSS_*` and `CARGO_CFG_TARGET_*`) @@ -1646,6 +1665,18 @@ pub fn find_interpreter() -> Result { } } +/// Locates and extracts the build host Python interpreter configuration. +/// +/// Lowers the configured Python version to `abi3_version` if required. +fn get_host_interpreter(abi3_version: Option) -> Result { + let interpreter_path = find_interpreter()?; + + let mut interpreter_config = InterpreterConfig::from_interpreter(interpreter_path)?; + interpreter_config.fixup_for_abi3_version(abi3_version)?; + + Ok(interpreter_config) +} + /// Generates an interpreter config suitable for cross-compilation. /// /// This must be called from PyO3's build script, because it relies on environment variables such as @@ -1666,26 +1697,40 @@ pub fn make_cross_compile_config() -> Result> { /// Only used by `pyo3-build-config` build script. #[allow(dead_code, unused_mut)] pub fn make_interpreter_config() -> Result { + let host = Triple::host(); let abi3_version = get_abi3_version(); + // See if we can safely skip the Python interpreter configuration detection. + // Unix "abi3" extension modules can usually be built without any interpreter. + let need_interpreter = abi3_version.is_none() || require_libdir_for_target(&host); + if have_python_interpreter() { - let mut interpreter_config = InterpreterConfig::from_interpreter(find_interpreter()?)?; - interpreter_config.fixup_for_abi3_version(abi3_version)?; - Ok(interpreter_config) - } else if let Some(version) = abi3_version { - let host = Triple::host(); - let mut interpreter_config = default_abi3_config(&host, version); - - // Auto generate python3.dll import libraries for Windows targets. - #[cfg(feature = "python3-dll-a")] - { - interpreter_config.lib_dir = self::abi3_import_lib::generate_abi3_import_lib(&host)?; + match get_host_interpreter(abi3_version) { + Ok(interpreter_config) => return Ok(interpreter_config), + // Bail if the interpreter configuration is required to build. + Err(e) if need_interpreter => return Err(e), + _ => { + // Fall back to the "abi3" defaults just as if `PYO3_NO_PYTHON` + // environment variable was set. + warn!("Compiling without a working Python interpreter."); + } } - - Ok(interpreter_config) } else { - bail!("An abi3-py3* feature must be specified when compiling without a Python interpreter.") + ensure!( + abi3_version.is_some(), + "An abi3-py3* feature must be specified when compiling without a Python interpreter." + ); + }; + + let mut interpreter_config = default_abi3_config(&host, abi3_version.unwrap()); + + // Auto generate python3.dll import libraries for Windows targets. + #[cfg(feature = "python3-dll-a")] + { + interpreter_config.lib_dir = self::abi3_import_lib::generate_abi3_import_lib(&host)?; } + + Ok(interpreter_config) } #[cfg(test)]