From 479fe6781feb0a06549b2a2b586f00aea02d3278 Mon Sep 17 00:00:00 2001 From: Sergey Kvachonok Date: Wed, 6 Apr 2022 17:41:57 +0300 Subject: [PATCH 1/9] pyo3-build-config: Add `python3-dll-a` crate support Automatically generate `python3.dll` import libraries for Windows compile targets in the build script. Adds a new PyO3 crate feature `generate-abi3-import-lib` enabling automatic import library generation. Closes #2231 --- Cargo.toml | 3 ++ pyo3-build-config/Cargo.toml | 2 + pyo3-build-config/src/abi3_import_lib.rs | 48 ++++++++++++++++++++++++ pyo3-build-config/src/impl_.rs | 29 ++++++++++++-- 4 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 pyo3-build-config/src/abi3_import_lib.rs diff --git a/Cargo.toml b/Cargo.toml index 7cc0f413efc..99655923824 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,9 @@ abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38", "pyo3-ffi/abi3-py38"] abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py39"] abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py310"] +# Automatically generates `python3.dll` import libraries for Windows targets. +generate-abi3-import-lib = ["pyo3-build-config/python3-dll-a"] + # Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the # Python interpreter if needed. auto-initialize = [] diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 16d189622cb..86ffa36e101 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -12,9 +12,11 @@ edition = "2018" [dependencies] once_cell = "1" +python3-dll-a = { version = "0.2", optional = true } target-lexicon = "0.12" [build-dependencies] +python3-dll-a = { version = "0.2", optional = true } target-lexicon = "0.12" [features] diff --git a/pyo3-build-config/src/abi3_import_lib.rs b/pyo3-build-config/src/abi3_import_lib.rs new file mode 100644 index 00000000000..2a089fbadfe --- /dev/null +++ b/pyo3-build-config/src/abi3_import_lib.rs @@ -0,0 +1,48 @@ +//! Optional `python3.dll` import library generator for Windows + +use std::env; +use std::path::PathBuf; + +use python3_dll_a::generate_implib_for_target; + +use crate::errors::{Context, Result}; + +use super::{Architecture, OperatingSystem, Triple}; + +/// Generates the `python3.dll` import library for Windows targets. +/// +/// Places the generated import library into the build script output directory +/// and returns the full library directory path. +/// +/// Does nothing if the target OS is not Windows. +pub(super) fn generate_abi3_import_lib(target: &Triple) -> Result> { + if target.operating_system != OperatingSystem::Windows { + return Ok(None); + } + + let out_dir = env::var_os("OUT_DIR") + .expect("generate_abi3_import_lib() must be called from a build script"); + + // Put the newly created import library into the build script output directory. + let mut out_lib_dir = PathBuf::from(out_dir); + out_lib_dir.push("lib"); + + // Convert `Architecture` enum to rustc `target_arch` option format. + let arch = match target.architecture { + // i686, i586, etc. + Architecture::X86_32(_) => "x86".to_string(), + other => other.to_string(), + }; + + let env = target.environment.to_string(); + + generate_implib_for_target(&out_lib_dir, &arch, &env) + .context("failed to generate python3.dll import library")?; + + let out_lib_dir_string = out_lib_dir + .to_str() + .ok_or("build directory is not a valid UTF-8 string")? + .to_owned(); + + Ok(Some(out_lib_dir_string)) +} diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 581004c4a90..d34cc7a1ac6 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -1,3 +1,11 @@ +//! Main implementation module included in both the `pyo3-build-config` library crate +//! and its build script. + +// Optional python3.dll import library generator for Windows +#[cfg(feature = "python3-dll-a")] +#[path = "abi3_import_lib.rs"] +mod abi3_import_lib; + use std::{ collections::{HashMap, HashSet}, convert::AsRef, @@ -1352,6 +1360,7 @@ fn cross_compile_from_sysconfigdata( /// Windows, macOS and Linux. /// /// Must be called from a PyO3 crate build script. +#[allow(unused_mut)] fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result { let version = cross_compile_config .version @@ -1381,7 +1390,13 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result Result> { /// Generates an interpreter config which will be hard-coded into the pyo3-build-config crate. /// Only used by `pyo3-build-config` build script. -#[allow(dead_code)] +#[allow(dead_code, unused_mut)] pub fn make_interpreter_config() -> Result { let abi3_version = get_abi3_version(); @@ -1659,7 +1674,15 @@ pub fn make_interpreter_config() -> Result { Ok(interpreter_config) } else if let Some(version) = abi3_version { let host = Triple::host(); - Ok(default_abi3_config(&host, version)) + let mut interpreter_config = default_abi3_config(&host, version); + + // Auto generate python3.dll import libraries for Windows targets. + #[cfg(feature = "python3-dll-a")] + { + interpreter_config.lib_dir = self::abi3_import_lib::generate_abi3_import_lib(&host)?; + } + + Ok(interpreter_config) } else { bail!("An abi3-py3* feature must be specified when compiling without a Python interpreter.") } From a7c0de36281c66f572cd25582791d3ddc51881d0 Mon Sep 17 00:00:00 2001 From: Sergey Kvachonok Date: Fri, 8 Apr 2022 12:18:17 +0300 Subject: [PATCH 2/9] Document `generate-abi3-import-lib` crate feature Update the user guide to describe its applicability to the native and cross-compilation build scenarios. --- Architecture.md | 8 ++++++- guide/src/building_and_distribution.md | 32 ++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/Architecture.md b/Architecture.md index d626fd9becc..d0e873f80a8 100644 --- a/Architecture.md +++ b/Architecture.md @@ -189,13 +189,19 @@ Some of the functionality of `pyo3-build-config`: See [#1123](https://github.com/PyO3/pyo3/pull/1123). - Cross-compiling configuration - If `TARGET` architecture and `HOST` architecture differ, we can find cross compile information - from environment variables (`PYO3_CROSS_LIB_DIR` and `PYO3_CROSS_PYTHON_VERSION`) or system files. + from environment variables (`PYO3_CROSS_LIB_DIR`, `PYO3_CROSS_PYTHON_VERSION` and + `PYO3_CROSS_PYTHON_IMPLEMENTATION`) or system files. When cross compiling extension modules it is often possible to make it work without any additional user input. + - When an experimental feature `generate-abi3-import-lib` is enabled, the `pyo3-ffi` build script can + generate `python3.dll` import libraries for Windows targets automatically via an external + [`python3-dll-a`] crate. This enables the users to cross compile abi3 extensions for Windows without + having to install any Windows Python libraries. [python/c api]: https://docs.python.org/3/c-api/ +[`python3-dll-a`]: https://docs.rs/python3-dll-a/latest/python3_dll_a/ diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index c4113ad15ac..0fda63b30fa 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -152,10 +152,20 @@ As your extension module may be run with multiple different Python versions you PyO3 is only able to link your extension module to api3 version up to and including your host Python version. E.g., if you set `abi3-py38` and try to compile the crate with a host of Python 3.7, the build will fail. -As an advanced feature, you can build PyO3 wheel without calling Python interpreter with the environment variable `PYO3_NO_PYTHON` set. On unix systems this works unconditionally; on Windows you must also set the `RUSTFLAGS` evironment variable to contain `-L native=/path/to/python/libs` so that the linker can find `python3.lib`. - > Note: If you set more that one of these api version feature flags the lowest version always wins. For example, with both `abi3-py37` and `abi3-py38` set, PyO3 would build a wheel which supports Python 3.7 and up. +#### Building `abi3` extensions without a Python interpreter + +As an advanced feature, you can build PyO3 wheel without calling Python interpreter with the environment variable `PYO3_NO_PYTHON` set. +On Unix-like systems this works unconditionally; on Windows you must also set the `RUSTFLAGS` environment variable +to contain `-L native=/path/to/python/libs` so that the linker can find `python3.lib`. + +If the `python3.dll` import library is not available, an experimental `generate-abi3-import-lib` crate +feature may be enabled, and the required library will be created and used by PyO3 automatically. + +*Note*: MSVC targets require LLVM binutils (`llvm-dlltool`) to be available in `PATH` for +the automatic import library generation feature to work. + #### Missing features Due to limitations in the Python API, there are a few `pyo3` features that do @@ -218,8 +228,8 @@ Thanks to Rust's great cross-compilation support, cross-compiling using PyO3 is * A toolchain for your target. * The appropriate options in your Cargo `.config` for the platform you're targeting and the toolchain you are using. -* A Python interpreter that's already been compiled for your target. -* A Python interpreter that is built for your host and available through the `PATH` or setting the [`PYO3_PYTHON`](#python-version) variable. +* A Python interpreter that's already been compiled for your target (optional when building "abi3" extension modules). +* A Python interpreter that is built for your host and available through the `PATH` or setting the [`PYO3_PYTHON`](#python-version) variable (optional when building "abi3" extension modules). After you've obtained the above, you can build a cross-compiled PyO3 module by using Cargo's `--target` flag. PyO3's build script will detect that you are attempting a cross-compile based on your host machine and the desired target. @@ -230,6 +240,14 @@ When cross-compiling, PyO3's build script cannot execute the target Python inter * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python installation. This variable is only needed if PyO3 cannot determine the version to target from `abi3-py3*` features, or if `PYO3_CROSS_LIB_DIR` is not set, or if there are multiple versions of Python present in `PYO3_CROSS_LIB_DIR`. * `PYO3_CROSS_PYTHON_IMPLEMENTATION`: Python implementation name ("CPython" or "PyPy") of the target Python installation. CPython is assumed by default when this variable is not set, unless `PYO3_CROSS_LIB_DIR` is set for a Unix-like target and PyO3 can get the interpreter configuration from `_sysconfigdata*.py`. +An experimental `pyo3` crate feature `generate-abi3-import-lib` enables the user to cross-compile +"abi3" extension modules for Windows targets without setting the `PYO3_CROSS_LIB_DIR` environment +variable or providing any Windows Python library files. It uses an external [`python3-dll-a`] crate +to generate import libraries for the Stable ABI Python DLL for MinGW-w64 and MSVC compile targets. +*Note*: MSVC targets require LLVM binutils to be available on the host system. +More specifically, `python3-dll-a` requires `llvm-dlltool` executable to be present in `PATH` when +targeting `*-pc-windows-msvc`. + An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`): ```sh @@ -255,7 +273,10 @@ cargo build --target x86_64-pc-windows-gnu ``` Any of the `abi3-py3*` features can be enabled instead of setting `PYO3_CROSS_PYTHON_VERSION` in the above examples. -`PYO3_CROSS_LIB_DIR` can often be omitted when cross compiling extension modules for Unix and macOS targets. + +`PYO3_CROSS_LIB_DIR` can often be omitted when cross compiling extension modules for Unix and macOS targets, +or when cross compiling "abi3" extension modules for Windows and the experimental `generate-abi3-import-lib` +crate feature is enabled. The following resources may also be useful for cross-compiling: - [github.com/japaric/rust-cross](https://github.com/japaric/rust-cross) is a primer on cross compiling Rust. @@ -267,3 +288,4 @@ The following resources may also be useful for cross-compiling: [`maturin`]: https://github.com/PyO3/maturin [`setuptools-rust`]: https://github.com/PyO3/setuptools-rust [PyOxidizer]: https://github.com/indygreg/PyOxidizer +[`python3-dll-a`]: https://docs.rs/python3-dll-a/latest/python3_dll_a/ From c84cecc1f7c61dfcf3015a88fd1ea44ded6c0434 Mon Sep 17 00:00:00 2001 From: Sergey Kvachonok Date: Fri, 8 Apr 2022 14:14:48 +0300 Subject: [PATCH 3/9] Add a ChangeLog entry for `generate-abi3-import-lib` --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53d9d87add3..066c13edb4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- Add an experimental `generate-abi3-import-lib` feature to auto-generate `python3.dll` import libraries for Windows. [#2282](https://github.com/PyO3/pyo3/pull/2282) + ### Changed - Default to "m" ABI tag when choosing `libpython` link name for CPython 3.7 on Unix. [#2288](https://github.com/PyO3/pyo3/pull/2288) From c5ca6d040b2009de5ad334ce8dc069a19a9008e6 Mon Sep 17 00:00:00 2001 From: messense Date: Sun, 10 Apr 2022 22:05:30 +0800 Subject: [PATCH 4/9] Add Windows abi3 wheel cross comiplation test --- .github/workflows/ci.yml | 13 +++++++++++++ examples/maturin-starter/Cargo.toml | 3 +++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae2409c73cf..01314b3adf3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -271,6 +271,19 @@ jobs: target: aarch64-apple-darwin args: --release -i python3.9 --no-sdist -m examples/maturin-starter/Cargo.toml + - name: Test cross compile to Windows + if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.10' }} + run: | + sudo apt-get install -y mingw-w64 + rustup target add x86_64-pc-windows-gnu + cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu + - name: Test cross compile to Windows with maturin + if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.10' }} + uses: messense/maturin-action@v1 + with: + target: x86_64-pc-windows-gnu + args: --no-sdist -m examples/maturin-starter/Cargo.toml --cargo-extra-args="--features abi3" + env: CARGO_TERM_VERBOSE: true CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }} diff --git a/examples/maturin-starter/Cargo.toml b/examples/maturin-starter/Cargo.toml index 94045488170..84d4dc35e59 100644 --- a/examples/maturin-starter/Cargo.toml +++ b/examples/maturin-starter/Cargo.toml @@ -10,4 +10,7 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { path = "../../", features = ["extension-module"] } +[features] +abi3 = ["pyo3/abi3-py37"] + [workspace] From eb6b6457e23a4201c750c16752169aa2584dee22 Mon Sep 17 00:00:00 2001 From: messense Date: Sun, 10 Apr 2022 22:11:04 +0800 Subject: [PATCH 5/9] Enable `generate-abi3-import-lib` feature for maturin-starter when abi3 feature is enabled --- examples/maturin-starter/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/maturin-starter/Cargo.toml b/examples/maturin-starter/Cargo.toml index 84d4dc35e59..21e8c90cc54 100644 --- a/examples/maturin-starter/Cargo.toml +++ b/examples/maturin-starter/Cargo.toml @@ -11,6 +11,6 @@ crate-type = ["cdylib"] pyo3 = { path = "../../", features = ["extension-module"] } [features] -abi3 = ["pyo3/abi3-py37"] +abi3 = ["pyo3/abi3-py37", "pyo3/generate-abi3-import-lib"] [workspace] From 8368c628586e0ce1a50c15ddb6b136fa7ac5c4ec Mon Sep 17 00:00:00 2001 From: messense Date: Sun, 10 Apr 2022 22:29:03 +0800 Subject: [PATCH 6/9] maturin still requires a Python interpreter on Windows for now Switch from Python 3.10 to Python 3.8 because I don't want to test with Rust 1.48.0 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01314b3adf3..9f2705eb771 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -272,17 +272,17 @@ jobs: args: --release -i python3.9 --no-sdist -m examples/maturin-starter/Cargo.toml - name: Test cross compile to Windows - if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.10' }} + if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.8' }} run: | sudo apt-get install -y mingw-w64 rustup target add x86_64-pc-windows-gnu cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu - name: Test cross compile to Windows with maturin - if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.10' }} + if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.8' }} uses: messense/maturin-action@v1 with: target: x86_64-pc-windows-gnu - args: --no-sdist -m examples/maturin-starter/Cargo.toml --cargo-extra-args="--features abi3" + args: -i python3.8 --no-sdist -m examples/maturin-starter/Cargo.toml --cargo-extra-args="--features abi3" env: CARGO_TERM_VERBOSE: true From 4ba11f76154e64778c4cfc456d170076703c53fc Mon Sep 17 00:00:00 2001 From: messense Date: Sun, 10 Apr 2022 23:16:04 +0800 Subject: [PATCH 7/9] Add a windows msvc cross compilation test --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f2705eb771..9999c65aa46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -273,10 +273,13 @@ jobs: - name: Test cross compile to Windows if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.8' }} + env: + CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld run: | - sudo apt-get install -y mingw-w64 - rustup target add x86_64-pc-windows-gnu + sudo apt-get install -y mingw-w64 llvm + rustup target add x86_64-pc-windows-gnu x86_64-pc-windows-msvc cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu + cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc - name: Test cross compile to Windows with maturin if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.8' }} uses: messense/maturin-action@v1 From 75ea17192768e532488815c55b81978ee1ce3532 Mon Sep 17 00:00:00 2001 From: messense Date: Mon, 11 Apr 2022 00:19:20 +0800 Subject: [PATCH 8/9] Use `cargo-xwin` to test windows msvc cross compilation --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9999c65aa46..bb2ef4353a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -274,12 +274,13 @@ jobs: - name: Test cross compile to Windows if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.8' }} env: - CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld + XWIN_ARCH: x86_64 run: | sudo apt-get install -y mingw-w64 llvm rustup target add x86_64-pc-windows-gnu x86_64-pc-windows-msvc + which cargo-xwin > /dev/null || cargo install cargo-xwin cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu - cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc + cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc - name: Test cross compile to Windows with maturin if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.8' }} uses: messense/maturin-action@v1 From 4347624bd36f31f79a9d48c8868601e33a0370c6 Mon Sep 17 00:00:00 2001 From: Sergey Kvachonok Date: Mon, 11 Apr 2022 09:56:52 +0300 Subject: [PATCH 9/9] Mention `generate-abi3-import-lib` in the features guide Update the feature list section of the user guide to include `generate-abi3-import-lib` description. --- guide/src/features.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/guide/src/features.md b/guide/src/features.md index 92d04815a10..d37059582da 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -30,6 +30,17 @@ These features are extensions of the `abi3` feature to specify the exact minimum See the [building and distribution](building_and_distribution.md#minimum-python-version-for-abi3) section for further detail. +### `generate-abi3-import-lib` + +This experimental feature is used to generate import libraries for the Stable ABI Python DLL +for MinGW-w64 and MSVC (cross-)compile targets. + +Enabling it allows to (cross-)compile `abi3` extension modules to any Windows targets +without having to install the Windows Python distribution files for the target. + +See the [building and distribution](building_and_distribution.md#building-abi3-extensions-without-a-python-interpreter) +section for further detail. + ## Features for embedding Python in Rust ### `auto-initialize`