From c49947f33521c1a00ad3ceebd6e8b82bbba17380 Mon Sep 17 00:00:00 2001 From: herquan <31046219+herquan@users.noreply.github.com> Date: Mon, 16 May 2022 19:50:51 -0700 Subject: [PATCH] Add macro append_to_inittab Sometimes we need to debug in a real environment with our module installed. `append_to_inittab` will be a wrapper for PyImport_AppendInittab (https://docs.python.org/3/c-api/import.html#c.PyImport_AppendInittab) and help us to do this --- CHANGELOG.md | 1 + guide/src/building_and_distribution.md | 4 ++++ pyo3-ffi/src/import.rs | 2 +- pyo3-macros-backend/src/module.rs | 3 ++- src/macros.rs | 23 ++++++++++++++++++ src/test_hygiene/misc.rs | 11 +++++++++ tests/test_append_to_inittab.rs | 32 ++++++++++++++++++++++++++ 7 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 tests/test_append_to_inittab.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a184d25e81..7cdf41919ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Implement `ToPyObject` for `[T; N]`. [#2313](https://github.com/PyO3/pyo3/pull/2313) - Added the internal `IntoPyResult` trait to give better error messages when function return types do not implement `IntoPy`. [#2326](https://github.com/PyO3/pyo3/pull/2326) - Add `PyDictKeys`, `PyDictValues` and `PyDictItems` Rust types to represent `dict_keys`, `dict_values` and `dict_items` types. [#2358](https://github.com/PyO3/pyo3/pull/2358) +- Add macro `append_to_inittab`. [#2377](https://github.com/PyO3/pyo3/pull/2377) ### Changed diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 60332c9ac6d..70cd4bcab40 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -225,6 +225,10 @@ The known complications are: If you encounter these or other complications when linking the interpreter statically, discuss them on [issue 416 on PyO3's Github](https://github.com/PyO3/pyo3/issues/416). It is hoped that eventually that discussion will contain enough information and solutions that PyO3 can offer first-class support for static embedding. +### Import your module when embedding the Python interpreter + +When you run your Rust binary with an embedded interpreter, any `#[pymodule]` created modules won't be accessible to import unless added to a table called `PyImport_Inittab` before the embedded interpreter is initialized. This will cause Python statements in your embedded interpreter such as `import your_new_module` to fail. You can call the macro [`append_to_inittab`]({{#PYO3_DOCS_URL}}/pyo3/macro.append_to_inittab.html) with your module before initializing the Python interpreter to add the module function into that table. (The Python interpreter will be initialized by calling `prepare_freethreaded_python`, `with_embedded_interpreter`, or `Python::with_gil` with the [`auto-initialize`](features.md#auto-initialize) feature enabled.) + ## Cross Compiling Thanks to Rust's great cross-compilation support, cross-compiling using PyO3 is relatively straightforward. To get started, you'll need a few pieces of software: diff --git a/pyo3-ffi/src/import.rs b/pyo3-ffi/src/import.rs index 794e0ee5480..e00843466e8 100644 --- a/pyo3-ffi/src/import.rs +++ b/pyo3-ffi/src/import.rs @@ -76,6 +76,6 @@ extern "C" { pub fn PyImport_AppendInittab( name: *const c_char, - initfunc: Option *mut PyObject>, + initfunc: Option *mut PyObject>, ) -> c_int; } diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 5cbd5f36630..65027dd34d1 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -78,6 +78,7 @@ pub fn pymodule_impl( #visibility mod #fnname { pub(crate) struct MakeDef; pub static DEF: #krate::impl_::pymodule::ModuleDef = MakeDef::make_def(); + pub const NAME: &'static str = concat!(stringify!(#name), "\0"); /// This autogenerated function is called by the python interpreter when importing /// the module. @@ -97,7 +98,7 @@ pub fn pymodule_impl( const fn make_def() -> impl_::ModuleDef { const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(#fnname); unsafe { - impl_::ModuleDef::new(concat!(stringify!(#name), "\0"), #doc, INITIALIZER) + impl_::ModuleDef::new(#fnname::NAME, #doc, INITIALIZER) } } } diff --git a/src/macros.rs b/src/macros.rs index 0a164b350f5..55fa542c0a9 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -150,3 +150,26 @@ macro_rules! wrap_pymodule { } }; } + +#[cfg(not(PyPy))] +/// Add the module to the initialization table in order to make embedded Python code to use it. +/// Module name is the argument. +/// +/// Use it before [`prepare_freethreaded_python`](crate::prepare_freethreaded_python) and +/// leave feature `auto-initialize` off +#[macro_export] +macro_rules! append_to_inittab { + ($module:ident) => { + unsafe { + ::std::assert_eq!( + $crate::ffi::Py_IsInitialized(), + 0, + "called `append_to_inittab` but a Python interpreter is already running." + ); + $crate::ffi::PyImport_AppendInittab( + $module::NAME.as_ptr() as *const ::std::os::raw::c_char, + ::std::option::Option::Some($module::init), + ); + } + }; +} diff --git a/src/test_hygiene/misc.rs b/src/test_hygiene/misc.rs index 1c0baaa2a39..5af7aeae5de 100644 --- a/src/test_hygiene/misc.rs +++ b/src/test_hygiene/misc.rs @@ -33,3 +33,14 @@ fn intern(py: crate::Python<'_>) { let _foo = crate::intern!(py, "foo"); let _bar = crate::intern!(py, stringify!(bar)); } + +#[allow(dead_code)] +fn append_to_inittab() { + #[crate::pymodule] + #[pyo3(crate = "crate")] + #[allow(clippy::unnecessary_wraps)] + fn module_for_inittab(_: crate::Python<'_>, _: &crate::types::PyModule) -> crate::PyResult<()> { + ::std::result::Result::Ok(()) + } + crate::append_to_inittab!(module_for_inittab); +} diff --git a/tests/test_append_to_inittab.rs b/tests/test_append_to_inittab.rs new file mode 100644 index 00000000000..220a73912f1 --- /dev/null +++ b/tests/test_append_to_inittab.rs @@ -0,0 +1,32 @@ +#![cfg(all(feature = "macros", not(PyPy)))] +use pyo3::prelude::*; + +#[pyfunction] +fn foo() -> usize { + 123 +} + +#[pymodule] +fn module_with_functions(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(foo, m)?).unwrap(); + Ok(()) +} + +#[cfg(not(PyPy))] +#[test] +fn test_module_append_to_inittab() { + use pyo3::append_to_inittab; + append_to_inittab!(module_with_functions); + Python::with_gil(|py| { + py.run( + r#" +import module_with_functions +assert module_with_functions.foo() == 123 +"#, + None, + None, + ) + .map_err(|e| e.print(py)) + .unwrap(); + }) +}