Add macro append_to_inittab (issue #2359)

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Implement `ToPyObject` for `[T; N]`. [#2313](https://github.com/PyO3/pyo3/pull/2313)
 - Added the internal `IntoPyResult` trait to give better error messages when function return types do not implement `IntoPy`. [#2326](https://github.com/PyO3/pyo3/pull/2326)
 - Add `PyDictKeys`, `PyDictValues` and `PyDictItems` Rust types to represent `dict_keys`, `dict_values` and `dict_items` types. [#2358](https://github.com/PyO3/pyo3/pull/2358)
+- Add macro `append_to_inittab`. [#2377](https://github.com/PyO3/pyo3/pull/2377)
 
 ### Changed
 

guide/src/building_and_distribution.md

@@ -225,6 +225,10 @@ The known complications are:
 
 If you encounter these or other complications when linking the interpreter statically, discuss them on [issue 416 on PyO3's Github](https://github.com/PyO3/pyo3/issues/416). It is hoped that eventually that discussion will contain enough information and solutions that PyO3 can offer first-class support for static embedding.
 
+### Import your module when embedding the Python interpreter
+
+When you run your Rust binary with an embedded interpreter, any `#[pymodule]` created modules won't be accessible to import unless added to a table called `PyImport_Inittab` before the embedded interpreter is initialized. This will cause Python statements in your embedded interpreter such as `import your_new_module` to fail. You can call the macro [`append_to_inittab`]({{#PYO3_DOCS_URL}}/pyo3/macro.append_to_inittab.html) with your module before initializing the Python interpreter to add the module function into that table. (The Python interpreter will be initialized by calling `prepare_freethreaded_python`, `with_embedded_interpreter`, or `Python::with_gil` with the [`auto-initialize`](features.md#auto-initialize) feature enabled.)
+
 ## Cross Compiling
 
 Thanks to Rust's great cross-compilation support, cross-compiling using PyO3 is relatively straightforward. To get started, you'll need a few pieces of software:

pyo3-ffi/src/import.rs

@@ -76,6 +76,6 @@ extern "C" {
 
     pub fn PyImport_AppendInittab(
         name: *const c_char,
-        initfunc: Option<extern "C" fn() -> *mut PyObject>,
+        initfunc: Option<unsafe extern "C" fn() -> *mut PyObject>,
     ) -> c_int;
 }

pyo3-macros-backend/src/module.rs

@@ -78,6 +78,7 @@ pub fn pymodule_impl(
         #visibility mod #fnname {
             pub(crate) struct MakeDef;
             pub static DEF: #krate::impl_::pymodule::ModuleDef = MakeDef::make_def();
+            pub const NAME: &'static str = concat!(stringify!(#name), "\0");
 
             /// This autogenerated function is called by the python interpreter when importing
             /// the module.
@@ -97,7 +98,7 @@ pub fn pymodule_impl(
                 const fn make_def() -> impl_::ModuleDef {
                     const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(#fnname);
                     unsafe {
-                        impl_::ModuleDef::new(concat!(stringify!(#name), "\0"), #doc, INITIALIZER)
+                        impl_::ModuleDef::new(#fnname::NAME, #doc, INITIALIZER)
                     }
                 }
             }

src/macros.rs

@@ -150,3 +150,26 @@ macro_rules! wrap_pymodule {
         }
     };
 }
+
+/// Add the module to the initialization table in order to make embedded Python code to use it.
+/// Module name is the argument.
+///
+/// Use it before [`prepare_freethreaded_python`](crate::prepare_freethreaded_python) and
+/// leave feature `auto-initialize` off
+#[cfg(not(PyPy))]
+#[macro_export]
+macro_rules! append_to_inittab {
+    ($module:ident) => {
+        unsafe {
+            if $crate::ffi::Py_IsInitialized() != 0 {
+                ::std::panic!(
+                    "called `append_to_inittab` but a Python interpreter is already running."
+                );
+            }
+            $crate::ffi::PyImport_AppendInittab(
+                $module::NAME.as_ptr() as *const ::std::os::raw::c_char,
+                ::std::option::Option::Some($module::init),
+            );
+        }
+    };
+}

src/test_hygiene/misc.rs

@@ -33,3 +33,15 @@ fn intern(py: crate::Python<'_>) {
     let _foo = crate::intern!(py, "foo");
     let _bar = crate::intern!(py, stringify!(bar));
 }
+
+#[allow(dead_code)]
+#[cfg(not(PyPy))]
+fn append_to_inittab() {
+    #[crate::pymodule]
+    #[pyo3(crate = "crate")]
+    #[allow(clippy::unnecessary_wraps)]
+    fn module_for_inittab(_: crate::Python<'_>, _: &crate::types::PyModule) -> crate::PyResult<()> {
+        ::std::result::Result::Ok(())
+    }
+    crate::append_to_inittab!(module_for_inittab);
+}

tests/test_append_to_inittab.rs

@@ -0,0 +1,32 @@
+#![cfg(all(feature = "macros", not(PyPy)))]
+use pyo3::prelude::*;
+
+#[pyfunction]
+fn foo() -> usize {
+    123
+}
+
+#[pymodule]
+fn module_with_functions(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
+    m.add_function(wrap_pyfunction!(foo, m)?).unwrap();
+    Ok(())
+}
+
+#[cfg(not(PyPy))]
+#[test]
+fn test_module_append_to_inittab() {
+    use pyo3::append_to_inittab;
+    append_to_inittab!(module_with_functions);
+    Python::with_gil(|py| {
+        py.run(
+            r#"
+import module_with_functions
+assert module_with_functions.foo() == 123
+"#,
+            None,
+            None,
+        )
+        .map_err(|e| e.print(py))
+        .unwrap();
+    })
+}