From c2cc186932c4a5d7d4f092cf29f18d53afa36cee Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 28 Dec 2020 16:02:57 +0000 Subject: [PATCH] auto-initialize: new feature to control initializing Python --- .github/workflows/ci.yml | 18 ++++---- Cargo.toml | 24 +++++++---- README.md | 5 ++- build.rs | 10 +++++ src/gil.rs | 91 +++++++++++++++++++++++++++++----------- src/lib.rs | 9 ++-- src/python.rs | 20 ++++++--- 7 files changed, 128 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9339677f6b1..d50312e7bd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,27 +85,28 @@ jobs: run: echo LD_LIBRARY_PATH=${pythonLocation}/lib >> $GITHUB_ENV - name: Build docs - run: cargo doc --features "num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }} + run: cargo doc --no-default-features --features "macros num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }} - - name: Build without default features + - name: Build (no features) run: cargo build --no-default-features --verbose --target ${{ matrix.platform.rust-target }} - - name: Build with default features - run: cargo build --features "num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }} + - name: Build (all additive features) + run: cargo build --no-default-features --features "macros num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }} # Run tests (except on PyPy, because no embedding API). - if: matrix.python-version != 'pypy-3.6' name: Test - run: cargo test --features "num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }} + run: cargo test --no-default-features --features "macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }} + # Run tests again, but in abi3 mode - if: matrix.python-version != 'pypy-3.6' name: Test (abi3) - run: cargo test --no-default-features --features "abi3,macros" --target ${{ matrix.platform.rust-target }} + run: cargo test --no-default-features --features "abi3 macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }} # Run tests again, for abi3-py36 (the minimal Python version) - if: (matrix.python-version != 'pypy-3.6') && (matrix.python-version != '3.6') name: Test (abi3-py36) - run: cargo test --no-default-features --features "abi3-py36,macros" --target ${{ matrix.platform.rust-target }} + run: cargo test --no-default-features --features "abi3-py36 macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }} - name: Test proc-macro code run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml --target ${{ matrix.platform.rust-target }} @@ -125,6 +126,9 @@ jobs: env: RUST_BACKTRACE: 1 RUSTFLAGS: "-D warnings" + # TODO: this is a hack to workaround compile_error! warnings about auto-initialize on PyPy + # Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this. + PYO3_CI: 1 coverage: needs: [fmt] diff --git a/Cargo.toml b/Cargo.toml index f4cdb94f813..cbdc2b2a541 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,26 +33,36 @@ assert_approx_eq = "1.1.0" trybuild = "1.0.23" rustversion = "1.0" proptest = { version = "0.10.1", default-features = false, features = ["std"] } +# features needed to run the PyO3 test suite +pyo3 = { path = ".", default-features = false, features = ["macros", "auto-initialize"] } [features] -default = ["macros"] -macros = ["ctor", "indoc", "inventory", "paste", "pyo3-macros", "unindent"] +default = ["macros", "auto-initialize"] + +# Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc. +macros = ["pyo3-macros", "ctor", "indoc", "inventory", "paste", "unindent"] + +# Use this feature when building an extension module. +# It tells the linker to keep the python symbols unresolved, +# so that the module can also be used with statically linked python interpreters. +extension-module = [] + # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more. abi3 = [] + # With abi3, we can manually set the minimum Python version. abi3-py36 = ["abi3-py37"] abi3-py37 = ["abi3-py38"] abi3-py38 = ["abi3-py39"] abi3-py39 = ["abi3"] +# Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the +# Python interpreter if needed. +auto-initialize = [] + # Optimizes PyObject to Vec conversion and so on. nightly = [] -# Use this feature when building an extension module. -# It tells the linker to keep the python symbols unresolved, -# so that the module can also be used with statically linked python interpreters. -extension-module = [] - [workspace] members = [ "pyo3-macros", diff --git a/README.md b/README.md index 5d4f552e0f9..124158dcbbc 100644 --- a/README.md +++ b/README.md @@ -104,8 +104,9 @@ If you want your Rust application to create a Python interpreter internally and use it to run Python code, add `pyo3` to your `Cargo.toml` like this: ```toml -[dependencies] -pyo3 = "0.13.0" +[dependencies.pyo3] +version = "0.13.0" +features = ["embedding", "auto-initialize"] ``` Example program displaying the value of `sys.version` and the current user name: diff --git a/build.rs b/build.rs index 7350c42de77..46135071734 100644 --- a/build.rs +++ b/build.rs @@ -749,6 +749,10 @@ fn configure(interpreter_config: &InterpreterConfig) -> Result<()> { } } + if interpreter_config.shared { + println!("cargo:rustc-cfg=Py_SHARED"); + } + if interpreter_config.version.implementation == PythonInterpreterKind::PyPy { println!("cargo:rustc-cfg=PyPy"); }; @@ -883,5 +887,11 @@ fn main() -> Result<()> { } } + // TODO: this is a hack to workaround compile_error! warnings about auto-initialize on PyPy + // Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this. + if env::var_os("PYO3_CI").is_some() { + println!("cargo:rustc-cfg=__pyo3_ci"); + } + Ok(()) } diff --git a/src/gil.rs b/src/gil.rs index 183af48ec0b..597d5a0ad0e 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -3,11 +3,11 @@ //! Interaction with python's global interpreter lock use crate::{ffi, internal_tricks::Unsendable, Python}; -use parking_lot::{const_mutex, Mutex}; +use parking_lot::{const_mutex, Mutex, Once}; use std::cell::{Cell, RefCell}; -use std::{mem::ManuallyDrop, ptr::NonNull, sync}; +use std::{mem::ManuallyDrop, ptr::NonNull}; -static START: sync::Once = sync::Once::new(); +static START: Once = Once::new(); thread_local! { /// This is a internal counter in pyo3 monitoring whether this thread has the GIL. @@ -45,16 +45,20 @@ pub(crate) fn gil_is_acquired() -> bool { /// If both the Python interpreter and Python threading are already initialized, /// this function has no effect. /// +/// # Availability +/// +/// This function is only available when linking against Python distributions that contain a +/// shared library. +/// +/// This function is not available on PyPy. +/// /// # Panic /// If the Python interpreter is initialized but Python threading is not, /// a panic occurs. /// It is not possible to safely access the Python runtime unless the main /// thread (the thread which originally initialized Python) also initializes /// threading. -/// -/// When writing an extension module, the `#[pymodule]` macro -/// will ensure that Python threading is initialized. -/// +#[cfg(all(Py_SHARED, not(PyPy)))] pub fn prepare_freethreaded_python() { // Protect against race conditions when Python is not yet initialized // and multiple threads concurrently call 'prepare_freethreaded_python()'. @@ -72,22 +76,18 @@ pub fn prepare_freethreaded_python() { // Note that the 'main thread' notion in Python isn't documented properly; // and running Python without one is not officially supported. - // PyPy does not support the embedding API - #[cfg(not(PyPy))] - { - ffi::Py_InitializeEx(0); - - // Make sure Py_Finalize will be called before exiting. - extern "C" fn finalize() { - unsafe { - if ffi::Py_IsInitialized() != 0 { - ffi::PyGILState_Ensure(); - ffi::Py_Finalize(); - } + ffi::Py_InitializeEx(0); + + // Make sure Py_Finalize will be called before exiting. + extern "C" fn finalize() { + unsafe { + if ffi::Py_IsInitialized() != 0 { + ffi::PyGILState_Ensure(); + ffi::Py_Finalize(); } } - libc::atexit(finalize); } + libc::atexit(finalize); // > Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have // > to call it yourself anymore. @@ -95,11 +95,10 @@ pub fn prepare_freethreaded_python() { if ffi::PyEval_ThreadsInitialized() == 0 { ffi::PyEval_InitThreads(); } - // PyEval_InitThreads() will acquire the GIL, - // but we don't want to hold it at this point + + // Py_InitializeEx() will acquire the GIL, but we don't want to hold it at this point // (it's not acquired in the other code paths) // So immediately release the GIL: - #[cfg(not(PyPy))] let _thread_state = ffi::PyEval_SaveThread(); // Note that the PyThreadState returned by PyEval_SaveThread is also held in TLS by the Python runtime, // and will be restored by PyGILState_Ensure. @@ -137,7 +136,51 @@ impl GILGuard { /// If PyO3 does not yet have a `GILPool` for tracking owned PyObject references, then this /// new `GILGuard` will also contain a `GILPool`. pub(crate) fn acquire() -> GILGuard { - prepare_freethreaded_python(); + // Maybe auto-initialize the GIL: + // - If auto-initialize feature set and supported, try to initalize the interpreter. + // - If the auto-initialize feature is set but unsupported, emit hard errors only when + // the extension-module feature is not activated - extension modules don't care about + // auto-initialize so this avoids breaking existing builds. + // - Otherwise, just check the GIL is initialized. + cfg_if::cfg_if! { + if #[cfg(all(feature = "auto-initialize", Py_SHARED, not(PyPy)))] { + prepare_freethreaded_python(); + } else if #[cfg(all(feature = "auto-initialize", not(feature = "extension-module"), not(Py_SHARED), not(__pyo3_ci)))] { + compile_error!(concat!( + "The `auto-initialize` feature is not supported when linking Python ", + "statically instead of with a shared library.\n\n", + "Please disable the `auto-initialize` feature, for example by entering the following ", + "in your cargo.toml:\n\n", + " pyo3 = { version = \"0.13.0\", default-features = false }\n\n", + "Alternatively, compile PyO3 using a Python distribution which contains a shared ", + "libary." + )); + } else if #[cfg(all(feature = "auto-initialize", not(feature = "extension-module"), PyPy, not(__pyo3_ci)))] { + compile_error!(concat!( + "The `auto-initialize` feature is not supported by PyPy.\n\n", + "Please disable the `auto-initialize` feature, for example by entering the following ", + "in your cargo.toml:\n\n", + " pyo3 = { version = \"0.13.0\", default-features = false }", + )); + } else { + // extension module feature enabled and PyPy or static linking + // OR auto-initialize feature not enabled + START.call_once_force(|_| unsafe { + // Use call_once_force because if there is a panic because the interpreter is not + // initialized, it's fine for the user to initialize the interpreter and retry. + assert_ne!( + ffi::Py_IsInitialized(), + 0, + "The Python interpreter is not initalized and the `auto-initialize` feature is not enabled." + ); + assert_ne!( + ffi::PyEval_ThreadsInitialized(), + 0, + "Python threading is not initalized and the `auto-initialize` feature is not enabled." + ); + }); + } + } let gstate = unsafe { ffi::PyGILState_Ensure() }; // acquire GIL diff --git a/src/lib.rs b/src/lib.rs index 634e72e9d2a..c71e5b82ec8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,8 +114,9 @@ //! Add `pyo3` to your `Cargo.toml`: //! //! ```toml -//! [dependencies] -//! pyo3 = "0.13.0" +//! [dependencies.pyo3] +//! version = "0.13.0" +//! features = ["embedding", "auto-initialize"] //! ``` //! //! Example program displaying the value of `sys.version`: @@ -145,12 +146,14 @@ pub use crate::conversion::{ ToBorrowedObject, ToPyObject, }; pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult}; +#[cfg(all(Py_SHARED, not(PyPy)))] +pub use crate::gil::prepare_freethreaded_python; pub use crate::gil::{GILGuard, GILPool}; pub use crate::instance::{Py, PyNativeType, PyObject}; pub use crate::pycell::{PyCell, PyRef, PyRefMut}; pub use crate::pyclass::PyClass; pub use crate::pyclass_init::PyClassInitializer; -pub use crate::python::{prepare_freethreaded_python, Python, PythonVersionInfo}; +pub use crate::python::{Python, PythonVersionInfo}; pub use crate::type_object::{type_flags, PyTypeInfo}; // Since PyAny is as important as PyObject, we expose it to the top level. pub use crate::types::PyAny; diff --git a/src/python.rs b/src/python.rs index b1f0668b049..90a7127ee36 100644 --- a/src/python.rs +++ b/src/python.rs @@ -11,8 +11,6 @@ use std::ffi::{CStr, CString}; use std::marker::PhantomData; use std::os::raw::{c_char, c_int}; -pub use gil::prepare_freethreaded_python; - /// Represents the major, minor, and patch (if any) versions of this interpreter. /// /// See [Python::version]. @@ -134,8 +132,13 @@ impl Python<'_> { /// Acquires the global interpreter lock, which allows access to the Python runtime. The /// provided closure F will be executed with the acquired `Python` marker token. /// - /// If the Python runtime is not already initialized, this function will initialize it. - /// See [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details. + /// If the `auto-initialize` feature is enabled and the Python runtime is not already + /// initialized, this function will initialize it. See + /// [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details. + /// + /// # Panics + /// - If the `auto-initialize` feature is not enabled and the Python interpreter is not + /// initialized. /// /// # Example /// ``` @@ -158,8 +161,9 @@ impl Python<'_> { impl<'p> Python<'p> { /// Acquires the global interpreter lock, which allows access to the Python runtime. /// - /// If the Python runtime is not already initialized, this function will initialize it. - /// See [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details. + /// If the `auto-initialize` feature is enabled and the Python runtime is not already + /// initialized, this function will initialize it. See + /// [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details. /// /// Most users should not need to use this API directly, and should prefer one of two options: /// 1. When implementing `#[pymethods]` or `#[pyfunction]` add a function argument @@ -172,6 +176,10 @@ impl<'p> Python<'p> { /// allowed, and will not deadlock. However, `GILGuard`s must be dropped in the reverse order /// to acquisition. If PyO3 detects this order is not maintained, it may be forced to begin /// an irrecoverable panic. + /// + /// # Panics + /// - If the `auto-initialize` feature is not enabled and the Python interpreter is not + /// initialized. #[inline] pub fn acquire_gil() -> GILGuard { GILGuard::acquire()