move ffi module to separate crate

Cargo.toml

@@ -19,6 +19,9 @@ cfg-if = "1.0"
 libc = "0.2.62"
 parking_lot = "0.11.0"
 
+# ffi bindings to the python interpreter, split into a seperate crate so they can be used independently
+pyo3-ffi = { path = "pyo3-ffi", version = "=0.15.1" }
+
 # support crates for macros feature
 pyo3-macros = { path = "pyo3-macros", version = "=0.15.1", optional = true }
 indoc = { version = "1.0.3", optional = true }
@@ -60,16 +63,16 @@ multiple-pymethods = ["inventory", "pyo3-macros/multiple-pymethods"]
 # Use this feature when building an extension module.
 # It tells the linker to keep the python symbols unresolved,
 # so that the module can also be used with statically linked python interpreters.
-extension-module = []
+extension-module = ["pyo3-ffi/extension-module"]
 
 # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
-abi3 = ["pyo3-build-config/abi3"]
+abi3 = ["pyo3-build-config/abi3", "pyo3-ffi/abi3"]
 
 # With abi3, we can manually set the minimum Python version.
-abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37"]
-abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38"]
-abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"]
-abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310"]
+abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37", "pyo3-ffi/abi3-py37"]
+abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38", "pyo3-ffi/abi3-py37"]
+abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py37"]
+abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py37"]
 
 # Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the
 # Python interpreter if needed.
@@ -126,6 +129,7 @@ harness = false
 
 [workspace]
 members = [
+    "pyo3-ffi",
     "pyo3-macros",
     "pyo3-macros-backend",
     "pytests/pyo3-benchmarks",

pyo3-ffi/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "pyo3-ffi"
+version = "0.15.1"
+description = "Python-API bindings for the PyO3 ecosystem"
+authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
+keywords = ["pyo3", "python", "cpython", "ffi"]
+homepage = "https://github.com/pyo3/pyo3"
+repository = "https://github.com/pyo3/pyo3"
+categories = ["api-bindings", "development-tools::ffi"]
+license = "Apache-2.0"
+edition = "2018"
+
+[dependencies]
+libc = "0.2.62"
+
+[features]
+
+default = []
+
+# Use this feature when building an extension module.
+# It tells the linker to keep the python symbols unresolved,
+# so that the module can also be used with statically linked python interpreters.
+extension-module = []
+
+# Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
+abi3 = ["pyo3-build-config/abi3"]
+
+# With abi3, we can manually set the minimum Python version.
+abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37"]
+abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38"]
+abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"]
+abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310"]
+
+
+
+[build-dependencies]
+pyo3-build-config = { path = "../pyo3-build-config", version = "0.15.1", features = ["resolve-config"] }
+
+

pyo3-ffi/build.rs

@@ -0,0 +1,151 @@
+use std::env;
+
+use pyo3_build_config::{
+    bail, ensure,
+    pyo3_build_script_impl::{
+        cargo_env_var, env_var, errors::Result, resolve_interpreter_config, InterpreterConfig,
+        PythonVersion,
+    },
+};
+
+/// Minimum Python version PyO3 supports.
+const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 6 };
+
+fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> {
+    ensure!(
+        interpreter_config.version >= MINIMUM_SUPPORTED_VERSION,
+        "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})",
+        interpreter_config.version,
+        MINIMUM_SUPPORTED_VERSION,
+    );
+
+    Ok(())
+}
+
+fn ensure_target_pointer_width(interpreter_config: &InterpreterConfig) -> Result<()> {
+    if let Some(pointer_width) = interpreter_config.pointer_width {
+        // Try to check whether the target architecture matches the python library
+        let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH")
+            .unwrap()
+            .as_str()
+        {
+            "64" => 64,
+            "32" => 32,
+            x => bail!("unexpected Rust target pointer width: {}", x),
+        };
+
+        ensure!(
+            rust_target == pointer_width,
+            "your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)",
+            rust_target,
+            pointer_width
+        );
+    }
+    Ok(())
+}
+
+fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> {
+    if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() {
+        if !interpreter_config.shared {
+            bail!(
+                "The `auto-initialize` feature is enabled, but your python installation only supports \
+                embedding the Python interpreter statically. If you are attempting to run tests, or a \
+                binary which is okay to link dynamically, install a Python distribution which ships \
+                with the Python shared library.\n\
+                \n\
+                Embedding the Python interpreter statically does not yet have first-class support in \
+                PyO3. If you are sure you intend to do this, disable the `auto-initialize` feature.\n\
+                \n\
+                For more information, see \
+                https://pyo3.rs/v{pyo3_version}/\
+                    building_and_distribution.html#embedding-python-in-rust",
+                pyo3_version = env::var("CARGO_PKG_VERSION").unwrap()
+            );
+        }
+
+        // TODO: PYO3_CI env is a hack to workaround CI with PyPy, where the `dev-dependencies`
+        // currently cause `auto-initialize` to be enabled in CI.
+        // Once MSRV is 1.51 or higher, use cargo's `resolver = "2"` instead.
+        if interpreter_config.implementation.is_pypy() && env::var_os("PYO3_CI").is_none() {
+            bail!("the `auto-initialize` feature is not supported with PyPy");
+        }
+    }
+    Ok(())
+}
+
+fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> {
+    let target_os = cargo_env_var("CARGO_CFG_TARGET_OS").unwrap();
+    let is_extension_module = cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some();
+    if target_os == "windows" || target_os == "android" || !is_extension_module {
+        // windows and android - always link
+        // other systems - only link if not extension module
+        println!(
+            "cargo:rustc-link-lib={link_model}{alias}{lib_name}",
+            link_model = if interpreter_config.shared {
+                ""
+            } else {
+                "static="
+            },
+            alias = if target_os == "windows" {
+                "pythonXY:"
+            } else {
+                ""
+            },
+            lib_name = interpreter_config.lib_name.as_ref().ok_or(
+                "attempted to link to Python shared library but config does not contain lib_name"
+            )?,
+        );
+        if let Some(lib_dir) = &interpreter_config.lib_dir {
+            println!("cargo:rustc-link-search=native={}", lib_dir);
+        }
+    }
+
+    Ok(())
+}
+
+/// Prepares the PyO3 crate for compilation.
+///
+/// This loads the config from pyo3-build-config and then makes some additional checks to improve UX
+/// for users.
+///
+/// Emits the cargo configuration based on this config as well as a few checks of the Rust compiler
+/// version to enable features which aren't supported on MSRV.
+fn configure_pyo3() -> Result<()> {
+    let interpreter_config = resolve_interpreter_config()?;
+
+    if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") {
+        print_config_and_exit(&interpreter_config);
+    }
+
+    ensure_python_version(&interpreter_config)?;
+    ensure_target_pointer_width(&interpreter_config)?;
+    ensure_auto_initialize_ok(&interpreter_config)?;
+
+    if !interpreter_config.suppress_build_script_link_lines {
+        emit_link_config(&interpreter_config)?;
+    }
+
+    interpreter_config.emit_pyo3_cfgs();
+
+    // Extra lines come last, to support last write wins.
+    for line in &interpreter_config.extra_build_script_lines {
+        println!("{}", line);
+    }
+
+    Ok(())
+}
+
+fn print_config_and_exit(config: &InterpreterConfig) {
+    println!("\n-- PYO3_PRINT_CONFIG=1 is set, printing configuration and halting compile --");
+    config
+        .to_writer(&mut std::io::stdout())
+        .expect("failed to print config to stdout");
+    std::process::exit(101);
+}
+
+fn main() {
+    if let Err(e) = configure_pyo3() {
+        eprintln!("error: {}", e.report());
+        std::process::exit(1)
+    }
+}

src/ffi/abstract_.rs → pyo3-ffi/src/abstract_.rs

@@ -1,5 +1,5 @@
-use crate::ffi::object::*;
-use crate::ffi::pyport::Py_ssize_t;
+use crate::object::*;
+use crate::pyport::Py_ssize_t;
 use std::os::raw::{c_char, c_int};
 use std::ptr;
 
@@ -96,10 +96,9 @@ extern "C" {
 #[cfg(not(any(Py_LIMITED_API, PyPy)))]
 #[inline]
 pub unsafe fn PyIter_Check(o: *mut PyObject) -> c_int {
-    (match (*crate::ffi::Py_TYPE(o)).tp_iternext {
+    (match (*crate::Py_TYPE(o)).tp_iternext {
         Some(tp_iternext) => {
-            tp_iternext as *const std::os::raw::c_void
-                != crate::ffi::_PyObject_NextNotImplemented as _
+            tp_iternext as *const std::os::raw::c_void != crate::_PyObject_NextNotImplemented as _
         }
         None => false,
     }) as c_int

src/ffi/bltinmodule.rs → pyo3-ffi/src/bltinmodule.rs

@@ -1,4 +1,4 @@
-use crate::ffi::object::PyTypeObject;
+use crate::object::PyTypeObject;
 
 #[cfg_attr(windows, link(name = "pythonXY"))]
 extern "C" {

src/ffi/boolobject.rs → pyo3-ffi/src/boolobject.rs

@@ -1,5 +1,5 @@
-use crate::ffi::longobject::PyLongObject;
-use crate::ffi::object::*;
+use crate::longobject::PyLongObject;
+use crate::object::*;
 use std::os::raw::{c_int, c_long};
 
 #[cfg_attr(windows, link(name = "pythonXY"))]

src/ffi/bytearrayobject.rs → pyo3-ffi/src/bytearrayobject.rs

@@ -1,5 +1,5 @@
-use crate::ffi::object::*;
-use crate::ffi::pyport::Py_ssize_t;
+use crate::object::*;
+use crate::pyport::Py_ssize_t;
 use std::os::raw::{c_char, c_int};
 
 #[cfg_attr(windows, link(name = "pythonXY"))]

src/ffi/bytesobject.rs → pyo3-ffi/src/bytesobject.rs

@@ -1,5 +1,5 @@
-use crate::ffi::object::*;
-use crate::ffi::pyport::Py_ssize_t;
+use crate::object::*;
+use crate::pyport::Py_ssize_t;
 use std::os::raw::{c_char, c_int};
 
 #[cfg_attr(windows, link(name = "pythonXY"))]

src/ffi/ceval.rs → pyo3-ffi/src/ceval.rs

@@ -1,5 +1,5 @@
-use crate::ffi::object::PyObject;
-use crate::ffi::pystate::PyThreadState;
+use crate::object::PyObject;
+use crate::pystate::PyThreadState;
 use std::os::raw::{c_char, c_int, c_void};
 
 extern "C" {
@@ -37,7 +37,7 @@ extern "C" {
     pub fn PyEval_GetGlobals() -> *mut PyObject;
     #[cfg_attr(PyPy, link_name = "PyPyEval_GetLocals")]
     pub fn PyEval_GetLocals() -> *mut PyObject;
-    pub fn PyEval_GetFrame() -> *mut crate::ffi::PyFrameObject;
+    pub fn PyEval_GetFrame() -> *mut crate::PyFrameObject;
     #[cfg_attr(PyPy, link_name = "PyPy_AddPendingCall")]
     pub fn Py_AddPendingCall(
         func: Option<extern "C" fn(arg1: *mut c_void) -> c_int>,
@@ -59,8 +59,8 @@ extern "C" {
     pub fn PyEval_GetFuncName(arg1: *mut PyObject) -> *const c_char;
     pub fn PyEval_GetFuncDesc(arg1: *mut PyObject) -> *const c_char;
     pub fn PyEval_GetCallStats(arg1: *mut PyObject) -> *mut PyObject;
-    pub fn PyEval_EvalFrame(arg1: *mut crate::ffi::PyFrameObject) -> *mut PyObject;
-    pub fn PyEval_EvalFrameEx(f: *mut crate::ffi::PyFrameObject, exc: c_int) -> *mut PyObject;
+    pub fn PyEval_EvalFrame(arg1: *mut crate::PyFrameObject) -> *mut PyObject;
+    pub fn PyEval_EvalFrameEx(f: *mut crate::PyFrameObject, exc: c_int) -> *mut PyObject;
     #[cfg_attr(PyPy, link_name = "PyPyEval_SaveThread")]
     pub fn PyEval_SaveThread() -> *mut PyThreadState;
     #[cfg_attr(PyPy, link_name = "PyPyEval_RestoreThread")]

src/ffi/codecs.rs → pyo3-ffi/src/codecs.rs

@@ -1,4 +1,4 @@
-use crate::ffi::object::PyObject;
+use crate::object::PyObject;
 use std::os::raw::{c_char, c_int};
 
 extern "C" {

src/ffi/complexobject.rs → pyo3-ffi/src/complexobject.rs

@@ -1,4 +1,4 @@
-use crate::ffi::object::*;
+use crate::object::*;
 use std::os::raw::{c_double, c_int};
 
 #[repr(C)]

src/ffi/context.rs → pyo3-ffi/src/context.rs

@@ -1,4 +1,4 @@
-use crate::ffi::object::{PyObject, PyTypeObject, Py_TYPE};
+use crate::object::{PyObject, PyTypeObject, Py_TYPE};
 use std::os::raw::{c_char, c_int};
 
 extern "C" {

src/ffi/cpython/abstract_.rs → pyo3-ffi/src/cpython/abstract_.rs

@@ -1,8 +1,8 @@
-use crate::ffi::{PyObject, Py_buffer, Py_ssize_t};
+use crate::{PyObject, Py_buffer, Py_ssize_t};
 use std::os::raw::{c_char, c_int, c_void};
 
 #[cfg(all(Py_3_8, not(PyPy)))]
-use crate::ffi::{
+use crate::{
     pyport::PY_SSIZE_T_MAX, vectorcallfunc, PyCallable_Check, PyThreadState, PyThreadState_GET,
     PyTuple_Check, PyType_HasFeature, Py_TPFLAGS_HAVE_VECTORCALL,
 };
@@ -51,7 +51,7 @@ pub unsafe fn PyVectorcall_NARGS(n: size_t) -> Py_ssize_t {
 #[inline(always)]
 pub unsafe fn PyVectorcall_Function(callable: *mut PyObject) -> Option<vectorcallfunc> {
     assert!(!callable.is_null());
-    let tp = crate::ffi::Py_TYPE(callable);
+    let tp = crate::Py_TYPE(callable);
     if PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL) == 0 {
         return None;
     }
@@ -218,7 +218,7 @@ extern "C" {
 #[cfg(not(any(Py_3_9, PyPy)))]
 #[inline]
 pub unsafe fn PyObject_CheckBuffer(o: *mut PyObject) -> c_int {
-    let tp_as_buffer = (*crate::ffi::Py_TYPE(o)).tp_as_buffer;
+    let tp_as_buffer = (*crate::Py_TYPE(o)).tp_as_buffer;
     (!tp_as_buffer.is_null() && (*tp_as_buffer).bf_getbuffer.is_some()) as c_int
 }
 

src/ffi/cpython/bytesobject.rs → pyo3-ffi/src/cpython/bytesobject.rs

@@ -1,5 +1,5 @@
-use crate::ffi::object::PyObject;
-use crate::ffi::pyport::Py_ssize_t;
+use crate::object::PyObject;
+use crate::pyport::Py_ssize_t;
 use std::os::raw::c_int;
 
 extern "C" {

src/ffi/cpython/ceval.rs → pyo3-ffi/src/cpython/ceval.rs

@@ -1,12 +1,9 @@
-use crate::ffi::cpython::pystate::Py_tracefunc;
-use crate::ffi::object::{freefunc, PyObject};
+use crate::cpython::pystate::Py_tracefunc;
+use crate::object::{freefunc, PyObject};
 use std::os::raw::c_int;
 
 extern "C" {
-    pub fn _PyEval_EvalFrameDefault(
-        arg1: *mut crate::ffi::PyFrameObject,
-        exc: c_int,
-    ) -> *mut PyObject;
+    pub fn _PyEval_EvalFrameDefault(arg1: *mut crate::PyFrameObject, exc: c_int) -> *mut PyObject;
     pub fn _PyEval_RequestCodeExtraIndex(func: freefunc) -> c_int;
     pub fn PyEval_SetProfile(trace_func: Option<Py_tracefunc>, arg1: *mut PyObject);
     pub fn PyEval_SetTrace(trace_func: Option<Py_tracefunc>, arg1: *mut PyObject);

src/ffi/cpython/code.rs → pyo3-ffi/src/cpython/code.rs

@@ -1,5 +1,5 @@
-use crate::ffi::object::*;
-use crate::ffi::pyport::Py_ssize_t;
+use crate::object::*;
+use crate::pyport::Py_ssize_t;
 use std::os::raw::{c_char, c_int, c_uchar, c_void};
 
 // skipped _Py_CODEUNIT
@@ -90,7 +90,7 @@ pub unsafe fn PyCode_Check(op: *mut PyObject) -> c_int {
 #[inline]
 #[cfg(not(PyPy))]
 pub unsafe fn PyCode_GetNumFree(op: *mut PyCodeObject) -> Py_ssize_t {
-    crate::ffi::PyTuple_GET_SIZE((*op).co_freevars)
+    crate::PyTuple_GET_SIZE((*op).co_freevars)
 }
 
 extern "C" {