Merge pull request #2375 from PyO3/release-0.16.5

release: 0.16.5

.github/workflows/ci.yml

@@ -177,7 +177,7 @@ jobs:
         run: |
           set -x
           cargo update -p indexmap --precise 1.6.2
-          cargo update -p hashbrown:0.12.0 --precise 0.9.1
+          cargo update -p hashbrown:0.12.1 --precise 0.9.1
           PROJECTS=("." "examples/decorator" "examples/maturin-starter" "examples/setuptools-rust-starter" "examples/word-count")
           for PROJ in ${PROJECTS[@]}; do
             cargo update --manifest-path "$PROJ/Cargo.toml" -p parking_lot --precise 0.11.0
@@ -276,11 +276,17 @@ jobs:
         env:
           XWIN_ARCH: x86_64
         run: |
+          set -ex
           sudo apt-get install -y mingw-w64 llvm
           rustup target add x86_64-pc-windows-gnu x86_64-pc-windows-msvc
           python -m pip install cargo-xwin
+          # abi3
           cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu
           cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc
+          # non-abi3
+          export PYO3_CROSS_PYTHON_VERSION=3.9
+          cargo build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-gnu
+          cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-msvc
       - name: Test cross compile to Windows with maturin
         if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.8' }}
         uses: messense/maturin-action@v1

Architecture.md

@@ -193,9 +193,9 @@ Some of the functionality of `pyo3-build-config`:
     `PYO3_CROSS_PYTHON_IMPLEMENTATION`) or system files.
     When cross compiling extension modules it is often possible to make it work without any
     additional user input.
-  - When an experimental feature `generate-abi3-import-lib` is enabled, the `pyo3-ffi` build script can
+  - When an experimental feature `generate-import-lib` is enabled, the `pyo3-ffi` build script can
     generate `python3.dll` import libraries for Windows targets automatically via an external
-    [`python3-dll-a`] crate. This enables the users to cross compile abi3 extensions for Windows without
+    [`python3-dll-a`] crate. This enables the users to cross compile Python extensions for Windows without
     having to install any Windows Python libraries.
 
 <!-- External Links -->

CHANGELOG.md

@@ -6,7 +6,21 @@ PyO3 versions, please see the [migration guide](https://pyo3.rs/latest/migration
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## Unreleased
+## [0.16.5] - 2022-05-15
+
+### Added
+
+- Add an experimental `generate-import-lib` feature to support auto-generating non-abi3 python import libraries for Windows targets. [#2364](https://github.com/PyO3/pyo3/pull/2364)
+- Add FFI definition `Py_ExitStatusException`. [#2374](https://github.com/PyO3/pyo3/pull/2374)
+
+### Changed
+
+- Deprecate experimental `generate-abi3-import-lib` feature in favor of the new `generate-import-lib` feature. [#2364](https://github.com/PyO3/pyo3/pull/2364)
+
+### Fixed
+
+- Added missing `warn_default_encoding` field to `PyConfig` on 3.10+. The previously missing field could result in incorrect behavior or crashes. [#2370](https://github.com/PyO3/pyo3/pull/2370)
+- Fixed order of `pathconfig_warnings` and `program_name` fields of `PyConfig` on 3.10+. Previously, the order of the fields was swapped and this could lead to incorrect behavior or crashes. [#2370](https://github.com/PyO3/pyo3/pull/2370)
 
 ## [0.16.4] - 2022-04-14
 
@@ -1151,7 +1165,8 @@ Yanked
 
 - Initial release
 
-[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.16.4...HEAD
+[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.16.5...HEAD
+[0.16.5]: https://github.com/pyo3/pyo3/compare/v0.16.4...v0.16.5
 [0.16.3]: https://github.com/pyo3/pyo3/compare/v0.16.3...v0.16.4
 [0.16.3]: https://github.com/pyo3/pyo3/compare/v0.16.2...v0.16.3
 [0.16.2]: https://github.com/pyo3/pyo3/compare/v0.16.1...v0.16.2

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pyo3"
-version = "0.16.4"
+version = "0.16.5"
 description = "Bindings to Python interpreter"
 authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
 readme = "README.md"
@@ -19,10 +19,10 @@ libc = "0.2.62"
 parking_lot = ">= 0.11, < 0.13"
 
 # ffi bindings to the python interpreter, split into a seperate crate so they can be used independently
-pyo3-ffi = { path = "pyo3-ffi", version = "=0.16.4" }
+pyo3-ffi = { path = "pyo3-ffi", version = "=0.16.5" }
 
 # support crates for macros feature
-pyo3-macros = { path = "pyo3-macros", version = "=0.16.4", optional = true }
+pyo3-macros = { path = "pyo3-macros", version = "=0.16.5", optional = true }
 indoc = { version = "1.0.3", optional = true }
 unindent = { version = "0.1.4", optional = true }
 
@@ -48,9 +48,10 @@ proptest = { version = "0.10.1", default-features = false, features = ["std"] }
 send_wrapper = "0.5"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.61"
+widestring = "0.5.1"
 
 [build-dependencies]
-pyo3-build-config = { path = "pyo3-build-config", version = "0.16.4", features = ["resolve-config"] }
+pyo3-build-config = { path = "pyo3-build-config", version = "0.16.5", features = ["resolve-config"] }
 
 [features]
 default = ["macros", "pyproto"]
@@ -67,7 +68,7 @@ pyproto = ["pyo3-macros/pyproto"]
 # Use this feature when building an extension module.
 # It tells the linker to keep the python symbols unresolved,
 # so that the module can also be used with statically linked python interpreters.
-extension-module = ["pyo3-build-config/extension-module", "pyo3-ffi/extension-module"]
+extension-module = ["pyo3-ffi/extension-module"]
 
 # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
 abi3 = ["pyo3-build-config/abi3", "pyo3-ffi/abi3", "pyo3-macros/abi3"]
@@ -79,7 +80,9 @@ abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py39"]
 abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py310"]
 
 # Automatically generates `python3.dll` import libraries for Windows targets.
-generate-abi3-import-lib = ["pyo3-build-config/python3-dll-a"]
+generate-import-lib = ["pyo3-ffi/generate-import-lib"]
+# Deprecated, replaced by `generate-import-lib`
+generate-abi3-import-lib = ["generate-import-lib"]
 
 # Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the
 # Python interpreter if needed.

README.md

@@ -68,7 +68,7 @@ name = "string_sum"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3 = { version = "0.16.4", features = ["extension-module"] }
+pyo3 = { version = "0.16.5", features = ["extension-module"] }
 ```
 
 **`src/lib.rs`**
@@ -132,7 +132,7 @@ Start a new project with `cargo new` and add  `pyo3` to the `Cargo.toml` like th
 
 ```toml
 [dependencies.pyo3]
-version = "0.16.4"
+version = "0.16.5"
 features = ["auto-initialize"]
 ```
 

examples/Cargo.toml

@@ -5,7 +5,7 @@ publish = false
 edition = "2018"
 
 [dev-dependencies]
-pyo3 = { version = "0.16.4", path = "..", features = ["auto-initialize", "extension-module"] }
+pyo3 = { version = "0.16.5", path = "..", features = ["auto-initialize", "extension-module"] }
 
 [[example]]
 name = "decorator"

examples/decorator/.template/pre-script.rhai

@@ -1,4 +1,4 @@
-variable::set("PYO3_VERSION", "0.16.4");
+variable::set("PYO3_VERSION", "0.16.5");
 file::rename(".template/Cargo.toml", "Cargo.toml");
 file::rename(".template/pyproject.toml", "pyproject.toml");
 file::rename(".template/tox.ini", "tox.ini");

examples/maturin-starter/.template/pre-script.rhai

@@ -1,4 +1,4 @@
-variable::set("PYO3_VERSION", "0.16.4");
+variable::set("PYO3_VERSION", "0.16.5");
 file::rename(".template/Cargo.toml", "Cargo.toml");
 file::rename(".template/pyproject.toml", "pyproject.toml");
 file::rename(".template/tox.ini", "tox.ini");

examples/maturin-starter/Cargo.toml

@@ -11,6 +11,7 @@ crate-type = ["cdylib"]
 pyo3 = { path = "../../", features = ["extension-module"] }
 
 [features]
-abi3 = ["pyo3/abi3-py37", "pyo3/generate-abi3-import-lib"]
+abi3 = ["pyo3/abi3-py37", "generate-import-lib"]
+generate-import-lib = ["pyo3/generate-import-lib"]
 
 [workspace]

examples/setuptools-rust-starter/.template/pre-script.rhai

@@ -1,4 +1,4 @@
-variable::set("PYO3_VERSION", "0.16.4");
+variable::set("PYO3_VERSION", "0.16.5");
 file::rename(".template/Cargo.toml", "Cargo.toml");
 file::rename(".template/setup.cfg", "setup.cfg");
 file::rename(".template/tox.ini", "tox.ini");

examples/word-count/.template/pre-script.rhai

@@ -1,4 +1,4 @@
-variable::set("PYO3_VERSION", "0.16.4");
+variable::set("PYO3_VERSION", "0.16.5");
 file::rename(".template/Cargo.toml", "Cargo.toml");
 file::rename(".template/tox.ini", "tox.ini");
 file::delete(".template");

guide/src/building_and_distribution.md

@@ -162,7 +162,7 @@ PyO3 will still attempt to compile `abi3` extension modules after displaying a w
 On Unix-like systems this works unconditionally; on Windows you must also set the `RUSTFLAGS` environment variable
 to contain `-L native=/path/to/python/libs` so that the linker can find `python3.lib`.
 
-If the `python3.dll` import library is not available, an experimental `generate-abi3-import-lib` crate
+If the `python3.dll` import library is not available, an experimental `generate-import-lib` crate
 feature may be enabled, and the required library will be created and used by PyO3 automatically.
 
 *Note*: MSVC targets require LLVM binutils (`llvm-dlltool`) to be available in `PATH` for
@@ -242,12 +242,12 @@ When cross-compiling, PyO3's build script cannot execute the target Python inter
 * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python installation. This variable is only needed if PyO3 cannot determine the version to target from `abi3-py3*` features, or if `PYO3_CROSS_LIB_DIR` is not set, or if there are multiple versions of Python present in `PYO3_CROSS_LIB_DIR`.
 * `PYO3_CROSS_PYTHON_IMPLEMENTATION`: Python implementation name ("CPython" or "PyPy") of the target Python installation. CPython is assumed by default when this variable is not set, unless `PYO3_CROSS_LIB_DIR` is set for a Unix-like target and PyO3 can get the interpreter configuration from `_sysconfigdata*.py`.
 
-An experimental `pyo3` crate feature `generate-abi3-import-lib` enables the user to cross-compile
-"abi3" extension modules for Windows targets without setting the `PYO3_CROSS_LIB_DIR` environment
+An experimental `pyo3` crate feature `generate-import-lib` enables the user to cross-compile
+extension modules for Windows targets without setting the `PYO3_CROSS_LIB_DIR` environment
 variable or providing any Windows Python library files. It uses an external [`python3-dll-a`] crate
-to generate import libraries for the Stable ABI Python DLL for MinGW-w64 and MSVC compile targets.
-*Note*: MSVC targets require LLVM binutils to be available on the host system.
-More specifically, `python3-dll-a` requires `llvm-dlltool` executable to be present in `PATH` when
+to generate import libraries for the Python DLL for MinGW-w64 and MSVC compile targets.
+*Note*: MSVC targets require LLVM binutils or MSVC build tools to be available on the host system.
+More specifically, `python3-dll-a` requires `llvm-dlltool` or `lib.exe` executable to be present in `PATH` when
 targeting `*-pc-windows-msvc`.
 
 An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`):
@@ -277,7 +277,7 @@ cargo build --target x86_64-pc-windows-gnu
 Any of the `abi3-py3*` features can be enabled instead of setting `PYO3_CROSS_PYTHON_VERSION` in the above examples.
 
 `PYO3_CROSS_LIB_DIR` can often be omitted when cross compiling extension modules for Unix and macOS targets,
-or when cross compiling "abi3" extension modules for Windows and the experimental `generate-abi3-import-lib`
+or when cross compiling extension modules for Windows and the experimental `generate-import-lib`
 crate feature is enabled.
 
 The following resources may also be useful for cross-compiling:

guide/src/debugging.md

@@ -32,9 +32,16 @@ Run Valgrind with `valgrind --suppressions=valgrind-python.supp ./my-command --w
 
 ## Getting a stacktrace
 
-The best start to investigate a crash such as an segmentation fault is a backtrace.
+The best start to investigate a crash such as an segmentation fault is a backtrace. You can set `RUST_BACKTRACE=1` as an environment variable to get the stack trace on a `panic!`. Alternatively you can use a debugger such as `gdb` to explore the issue. Rust provides a wrapper, `rust-gdb`, which has pretty-printers for inspecting Rust variables. Since PyO3 uses `cdylib` for Python shared objects, it does not receive the pretty-print debug hooks in `rust-gdb` ([rust-lang/rust#96365](https://github.com/rust-lang/rust/issues/96365)). The mentioned issue contains a workaround for enabling pretty-printers in this case.
 
  * Link against a debug build of python as described in the previous chapter
- * Run `gdb <my-binary>`
+ * Run `rust-gdb <my-binary>`
+ * Set a breakpoint (`b`) on `rust_panic` if you are investigating a `panic!`
  * Enter `r` to run
  * After the crash occurred, enter `bt` or `bt full` to print the stacktrace
+
+ Often it is helpful to run a small piece of Python code to exercise a section of Rust.
+
+ ```console
+ rust-gdb --args python -c "import my_package; my_package.sum_to_string(1, 2)"
+ ```

guide/src/features.md

@@ -30,12 +30,12 @@ These features are extensions of the `abi3` feature to specify the exact minimum
 
 See the [building and distribution](building_and_distribution.md#minimum-python-version-for-abi3) section for further detail.
 
-### `generate-abi3-import-lib`
+### `generate-import-lib`
 
-This experimental feature is used to generate import libraries for the Stable ABI Python DLL
+This experimental feature is used to generate import libraries for Python DLL
 for MinGW-w64 and MSVC (cross-)compile targets.
 
-Enabling it allows to (cross-)compile `abi3` extension modules to any Windows targets
+Enabling it allows to (cross-)compile extension modules to any Windows targets
 without having to install the Windows Python distribution files for the target.
 
 See the [building and distribution](building_and_distribution.md#building-abi3-extensions-without-a-python-interpreter)

pyo3-build-config/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pyo3-build-config"
-version = "0.16.4"
+version = "0.16.5"
 description = "Build configuration for the PyO3 ecosystem"
 authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
 keywords = ["pyo3", "python", "cpython", "ffi"]
@@ -12,11 +12,11 @@ edition = "2018"
 
 [dependencies]
 once_cell = "1"
-python3-dll-a = { version = "0.2", optional = true }
+python3-dll-a = { version = "0.2.2", optional = true }
 target-lexicon = "0.12"
 
 [build-dependencies]
-python3-dll-a = { version = "0.2", optional = true }
+python3-dll-a = { version = "0.2.2", optional = true }
 target-lexicon = "0.12"
 
 [features]

pyo3-build-config/src/impl_.rs

@@ -3,8 +3,8 @@
 
 // Optional python3.dll import library generator for Windows
 #[cfg(feature = "python3-dll-a")]
-#[path = "abi3_import_lib.rs"]
-mod abi3_import_lib;
+#[path = "import_lib.rs"]
+mod import_lib;
 
 use std::{
     collections::{HashMap, HashSet},
@@ -456,6 +456,28 @@ print("mingw", get_platform().startswith("mingw"))
         })
     }
 
+    #[allow(clippy::unnecessary_wraps)]
+    pub fn fixup_import_libs(&mut self) -> Result<()> {
+        let target = target_triple_from_env();
+        if self.lib_name.is_none() && target.operating_system == OperatingSystem::Windows {
+            self.lib_name = Some(default_lib_name_windows(
+                self.version,
+                self.implementation,
+                self.abi3,
+                false,
+            ));
+        }
+        // Auto generate python3.dll import libraries for Windows targets.
+        #[cfg(feature = "python3-dll-a")]
+        {
+            if self.lib_dir.is_none() {
+                let py_version = if self.abi3 { None } else { Some(self.version) };
+                self.lib_dir = self::import_lib::generate_import_lib(&target, py_version)?;
+            }
+        }
+        Ok(())
+    }
+
     #[doc(hidden)]
     /// Serialize the `InterpreterConfig` and print it to the environment for Cargo to pass along
     /// to dependent packages during build time.
@@ -1391,13 +1413,11 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<In
         .unwrap_or(PythonImplementation::CPython);
 
     let lib_name = if cross_compile_config.target.operating_system == OperatingSystem::Windows {
-        let mingw = cross_compile_config.target.environment == Environment::Gnu;
-
         Some(default_lib_name_windows(
             version,
             implementation,
             abi3,
-            mingw,
+            false,
         ))
     } else if is_linking_libpython_for_target(&cross_compile_config.target) {
         Some(default_lib_name_unix(version, implementation, None))
@@ -1409,8 +1429,9 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<In
 
     // Auto generate python3.dll import libraries for Windows targets.
     #[cfg(feature = "python3-dll-a")]
-    if abi3 && lib_dir.is_none() {
-        lib_dir = self::abi3_import_lib::generate_abi3_import_lib(&cross_compile_config.target)?;
+    if lib_dir.is_none() {
+        let py_version = if abi3 { None } else { Some(version) };
+        lib_dir = self::import_lib::generate_import_lib(&cross_compile_config.target, py_version)?;
     }
 
     Ok(InterpreterConfig {
@@ -1723,7 +1744,12 @@ pub fn make_interpreter_config() -> Result<InterpreterConfig> {
     // Auto generate python3.dll import libraries for Windows targets.
     #[cfg(feature = "python3-dll-a")]
     {
-        interpreter_config.lib_dir = self::abi3_import_lib::generate_abi3_import_lib(&host)?;
+        let py_version = if interpreter_config.abi3 {
+            None
+        } else {
+            Some(interpreter_config.version)
+        };
+        interpreter_config.lib_dir = self::import_lib::generate_import_lib(&host, py_version)?;
     }
 
     Ok(interpreter_config)
@@ -2130,7 +2156,7 @@ mod tests {
                 version: PythonVersion { major: 3, minor: 8 },
                 shared: true,
                 abi3: false,
-                lib_name: Some("python3.8".into()),
+                lib_name: Some("python38".into()),
                 lib_dir: Some("/usr/lib/mingw".into()),
                 executable: None,
                 pointer_width: None,