Skip to content

Commit

Permalink
embedding: use Py_SHARED cfg flag
Browse files Browse the repository at this point in the history
  • Loading branch information
@davidhewitt
davidhewitt committed Jan 1, 2021
1 parent bcaab65 commit 9d4a410
Show file tree
Hide file tree
Showing 6 changed files with 101 additions and 67 deletions.
15 changes: 8 additions & 7 deletions .github/workflows/ci.yml
View file
Expand Up @@ -85,27 +85,28 @@ jobs:
run: echo LD_LIBRARY_PATH=${pythonLocation}/lib >> $GITHUB_ENV

- name: Build docs
run: cargo doc --features "num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
run: cargo doc --no-default-features --features "macros num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}

- name: Build without default features
- name: Build (no features)
run: cargo build --no-default-features --verbose --target ${{ matrix.platform.rust-target }}

- name: Build with default features
run: cargo build --features "num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
- name: Build (all additive features)
run: cargo build --no-default-features --features "macros num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}

# Run tests (except on PyPy, because no embedding API).
- if: matrix.python-version != 'pypy-3.6'
name: Test
run: cargo test --features "num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
run: cargo test --no-default-features --features "macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}

# Run tests again, but in abi3 mode
- if: matrix.python-version != 'pypy-3.6'
name: Test (abi3)
run: cargo test --no-default-features --features "abi3,macros" --target ${{ matrix.platform.rust-target }}
run: cargo test --no-default-features --features "abi3 macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}

# Run tests again, for abi3-py36 (the minimal Python version)
- if: (matrix.python-version != 'pypy-3.6') && (matrix.python-version != '3.6')
name: Test (abi3-py36)
run: cargo test --no-default-features --features "abi3-py36,macros" --target ${{ matrix.platform.rust-target }}
run: cargo test --no-default-features --features "abi3-py36 macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}

- name: Test proc-macro code
run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml --target ${{ matrix.platform.rust-target }}
Expand Down
9 changes: 3 additions & 6 deletions Cargo.toml
View file
Expand Up @@ -34,10 +34,10 @@ trybuild = "1.0.23"
rustversion = "1.0"
proptest = { version = "0.10.1", default-features = false, features = ["std"] }
# features needed to run the PyO3 test suite
pyo3 = { path = ".", default-features = false, features = ["macros", "embedding", "auto-initialize"] }
pyo3 = { path = ".", default-features = false, features = ["macros", "auto-initialize"] }

[features]
default = ["macros", "embedding", "auto-initialize"]
default = ["macros", "auto-initialize"]

# Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc.
macros = ["pyo3-macros", "ctor", "indoc", "inventory", "paste", "unindent"]
Expand All @@ -56,12 +56,9 @@ abi3-py37 = ["abi3-py38"]
abi3-py38 = ["abi3-py39"]
abi3-py39 = ["abi3"]

# Enables embedding a Python interpreter inside Rust programs.
embedding = []

# Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the
# Python interpreter if needed.
auto-initialize = ["embedding"]
auto-initialize = []

# Optimizes PyObject to Vec conversion and so on.
nightly = []
Expand Down
4 changes: 4 additions & 0 deletions build.rs
View file
Expand Up @@ -749,6 +749,10 @@ fn configure(interpreter_config: &InterpreterConfig) -> Result<()> {
}
}

if interpreter_config.shared {
println!("cargo:rustc-cfg=Py_SHARED");
}

if interpreter_config.version.implementation == PythonInterpreterKind::PyPy {
println!("cargo:rustc-cfg=PyPy");
};
Expand Down
137 changes: 84 additions & 53 deletions src/gil.rs
View file
Expand Up @@ -45,17 +45,20 @@ pub(crate) fn gil_is_acquired() -> bool {
/// If both the Python interpreter and Python threading are already initialized,
/// this function has no effect.
///
/// # Features
/// # Availability
///
/// This function is only available with the `embedding` feature.
/// This function is only available when linking against Python distributions that contain a
/// shared library.
///
/// This function is not available on PyPy.
///
/// # Panic
/// If the Python interpreter is initialized but Python threading is not,
/// a panic occurs.
/// It is not possible to safely access the Python runtime unless the main
/// thread (the thread which originally initialized Python) also initializes
/// threading.
#[cfg(feature = "embedding")]
#[cfg(all(Py_SHARED, not(PyPy)))]
pub fn prepare_freethreaded_python() {
// Protect against race conditions when Python is not yet initialized
// and multiple threads concurrently call 'prepare_freethreaded_python()'.
Expand All @@ -67,42 +70,38 @@ pub fn prepare_freethreaded_python() {
// as we can't make the existing Python main thread acquire the GIL.
assert_ne!(ffi::PyEval_ThreadsInitialized(), 0);
} else {
// TODO remove this cfg once build.rs rejects embedding feature misuse.
#[cfg(not(PyPy))]
{
// Initialize Python.
// We use Py_InitializeEx() with initsigs=0 to disable Python signal handling.
// Signal handling depends on the notion of a 'main thread', which doesn't exist in this case.
// Note that the 'main thread' notion in Python isn't documented properly;
// and running Python without one is not officially supported.

ffi::Py_InitializeEx(0);

// Make sure Py_Finalize will be called before exiting.
extern "C" fn finalize() {
unsafe {
if ffi::Py_IsInitialized() != 0 {
ffi::PyGILState_Ensure();
ffi::Py_Finalize();
}
}
}
libc::atexit(finalize);
// Initialize Python.
// We use Py_InitializeEx() with initsigs=0 to disable Python signal handling.
// Signal handling depends on the notion of a 'main thread', which doesn't exist in this case.
// Note that the 'main thread' notion in Python isn't documented properly;
// and running Python without one is not officially supported.

// > Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have
// > to call it yourself anymore.
#[cfg(not(Py_3_7))]
if ffi::PyEval_ThreadsInitialized() == 0 {
ffi::PyEval_InitThreads();
ffi::Py_InitializeEx(0);

// Make sure Py_Finalize will be called before exiting.
extern "C" fn finalize() {
unsafe {
if ffi::Py_IsInitialized() != 0 {
ffi::PyGILState_Ensure();
ffi::Py_Finalize();
}
}
}
libc::atexit(finalize);

// Py_InitializeEx() will acquire the GIL, but we don't want to hold it at this point
// (it's not acquired in the other code paths)
// So immediately release the GIL:
let _thread_state = ffi::PyEval_SaveThread();
// Note that the PyThreadState returned by PyEval_SaveThread is also held in TLS by the Python runtime,
// and will be restored by PyGILState_Ensure.
// > Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have
// > to call it yourself anymore.
#[cfg(not(Py_3_7))]
if ffi::PyEval_ThreadsInitialized() == 0 {
ffi::PyEval_InitThreads();
}

// Py_InitializeEx() will acquire the GIL, but we don't want to hold it at this point
// (it's not acquired in the other code paths)
// So immediately release the GIL:
let _thread_state = ffi::PyEval_SaveThread();
// Note that the PyThreadState returned by PyEval_SaveThread is also held in TLS by the Python runtime,
// and will be restored by PyGILState_Ensure.
}
});
}
Expand Down Expand Up @@ -137,24 +136,56 @@ impl GILGuard {
/// If PyO3 does not yet have a `GILPool` for tracking owned PyObject references, then this
/// new `GILGuard` will also contain a `GILPool`.
pub(crate) fn acquire() -> GILGuard {
#[cfg(feature = "auto-initialize")]
prepare_freethreaded_python();

#[cfg(not(feature = "auto-initialize"))]
START.call_once_force(|_| unsafe {
// Use call_once_force because if there is a panic because the interpreter is not
// initialized, it's fine for the user to initialize the interpreter and retry.
assert_ne!(
ffi::Py_IsInitialized(),
0,
"The Python interpreter is not initalized and the `auto-initialize` feature is not enabled."
);
assert_ne!(
ffi::PyEval_ThreadsInitialized(),
0,
"Python threading is not initalized and the `auto-initialize` feature is not enabled."
);
});
// Maybe auto-initialize the GIL:
// - If auto-initialize feature set and supported, try to initalize the interpreter.
// - If the auto-initialize feature is set but unsupported, emit hard errors only when
// the extension-module feature is not activated - extension modules don't care about
// auto-initialize so this avoids breaking existing builds.
// - Otherwise, just check the GIL is initialized.
cfg_if::cfg_if! {
if #[cfg(all(feature = "auto-initialize", Py_SHARED, not(PyPy)))] {
prepare_freethreaded_python();
} else if #[cfg(all(feature = "auto-initialize", not(feature = "extension-module"), not(Py_SHARED), not(rustdoc)))] {
compile_error!(concat!(
"The `auto-initialize` feature is not supported when linking Python ",
"statically instead of with a shared library.\n",
"\n",
"Please disable the `auto-initialize` feature, for example by entering the following ",
"in your cargo.toml:\n",
"\n",
" pyo3 = { version = \"0.13.0\", default-features = false }\n",
"\n",
"Alternatively, compile PyO3 using a Python distribution which contains a shared ",
"libary."
));
} else if #[cfg(all(feature = "auto-initialize", not(feature = "extension-module"), PyPy, not(rustdoc)))] {
compile_error!(concat!(
"The `auto-initialize` feature is not supported by PyPy.\n",
"\n",
"Please disable the `auto-initialize` feature, for example by entering the following ",
"in your cargo.toml:\n",
"\n",
" pyo3 = { version = \"0.13.0\", default-features = false }\n",
));
} else {
// extension module feature enabled and PyPy or static linking
// OR auto-initialize feature not enabled
START.call_once_force(|_| unsafe {
// Use call_once_force because if there is a panic because the interpreter is not
// initialized, it's fine for the user to initialize the interpreter and retry.
assert_ne!(
ffi::Py_IsInitialized(),
0,
"The Python interpreter is not initalized and the `auto-initialize` feature is not enabled."
);
assert_ne!(
ffi::PyEval_ThreadsInitialized(),
0,
"Python threading is not initalized and the `auto-initialize` feature is not enabled."
);
});
}
}

let gstate = unsafe { ffi::PyGILState_Ensure() }; // acquire GIL

Expand Down
2 changes: 1 addition & 1 deletion src/lib.rs
View file
Expand Up @@ -146,7 +146,7 @@ pub use crate::conversion::{
ToBorrowedObject, ToPyObject,
};
pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult};
#[cfg(feature = "embedding")]
#[cfg(all(Py_SHARED, not(PyPy)))]
pub use crate::gil::prepare_freethreaded_python;
pub use crate::gil::{GILGuard, GILPool};
pub use crate::instance::{Py, PyNativeType, PyObject};
Expand Down
1 change: 1 addition & 0 deletions src/python.rs
View file
Expand Up @@ -170,6 +170,7 @@ impl<'p> Python<'p> {
/// `py: Python` to receive access to the GIL context in which the function is running.
/// 2. Use [`Python::with_gil`](#method.with_gil) to run a closure with the GIL, acquiring
/// only if needed.
///
/// **Note:** This return type from this function, `GILGuard`, is implemented as a RAII guard
/// around the C-API Python_EnsureGIL. This means that multiple `acquire_gil()` calls are
/// allowed, and will not deadlock. However, `GILGuard`s must be dropped in the reverse order
Expand Down

0 comments on commit 9d4a410

Please sign in to comment.