implement ffi/cpython/pystate (#1687)

* dont reexport cpython items twice

* add ffi module level doc

* reorder to header file order

* cargo fmt

* implement cpython/pystate

* fix import errors

* make PyInterpreterState opaque

* update changelog

* fix formatting

CHANGELOG.md

@@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Add support for extracting `PathBuf` from `pathlib.Path`. [#1654](https://github.com/PyO3/pyo3/pull/1654)
 - Add `#[pyo3(text_signature = "...")]` syntax for setting text signature. [#1658](https://github.com/PyO3/pyo3/pull/1658)
 - Add support for setting and retrieving exception cause. [#1679](https://github.com/PyO3/pyo3/pull/1679)
+- Add FFI definitions from `cpython/pystate.h`.[#1687](https://github.com/PyO3/pyo3/pull/1687/)
 
 ### Changed
 
@@ -79,6 +80,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Remove `raw_pycfunction!` macro. [#1619](https://github.com/PyO3/pyo3/pull/1619)
 - Remove `PyClassAlloc` trait. [#1657](https://github.com/PyO3/pyo3/pull/1657)
 - Remove `PyList::get_parked_item`. [#1664](https://github.com/PyO3/pyo3/pull/1664)
+- Remove the layout of the ffi/ffi::cpython module from the public api [#1687](https://github.com/PyO3/pyo3/pull/1687/). If you were importing things with `pyo3::ffi::<module>::<item>;`, please use `pyo3::ffi::<item>;` instead
+
+
 
 ### Fixed
 

src/ffi/ceval.rs

@@ -2,10 +2,6 @@ use crate::ffi::object::PyObject;
 use crate::ffi::pystate::PyThreadState;
 use std::os::raw::{c_char, c_int, c_void};
 
-// TODO: move to cpython
-pub type _PyFrameEvalFunction =
-    extern "C" fn(*mut crate::ffi::PyFrameObject, c_int) -> *mut PyObject;
-
 extern "C" {
     #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))]
     #[cfg_attr(PyPy, link_name = "PyPyEval_CallObjectWithKeywords")]

src/ffi/cpython/mod.rs

@@ -1,23 +1,26 @@
-pub mod abstract_;
+pub(crate) mod abstract_;
 // skipped bytearrayobject.h
 #[cfg(not(PyPy))]
-pub mod bytesobject;
-pub mod ceval;
-pub mod code;
-pub mod compile;
+pub(crate) mod bytesobject;
+pub(crate) mod ceval;
+pub(crate) mod code;
+pub(crate) mod compile;
 #[cfg(not(PyPy))]
-pub mod dictobject;
+pub(crate) mod dictobject;
 // skipped fileobject.h
-pub mod frameobject;
-pub mod import;
+pub(crate) mod frameobject;
+pub(crate) mod import;
 #[cfg(all(Py_3_8, not(PyPy)))]
-pub mod initconfig;
+pub(crate) mod initconfig;
 // skipped interpreteridobject.h
-pub mod listobject;
-pub mod object;
-pub mod pydebug;
+pub(crate) mod listobject;
+pub(crate) mod object;
+pub(crate) mod pydebug;
 #[cfg(all(Py_3_8, not(PyPy)))]
-pub mod pylifecycle;
+pub(crate) mod pylifecycle;
+
+#[cfg(all(Py_3_8, not(PyPy)))]
+pub(crate) mod pystate;
 
 pub use self::abstract_::*;
 #[cfg(not(PyPy))]
@@ -36,3 +39,6 @@ pub use self::object::*;
 pub use self::pydebug::*;
 #[cfg(all(Py_3_8, not(PyPy)))]
 pub use self::pylifecycle::*;
+
+#[cfg(all(Py_3_8, not(PyPy)))]
+pub use self::pystate::*;

src/ffi/cpython/pystate.rs

@@ -0,0 +1,46 @@
+use crate::ffi::pystate::{PyInterpreterState, PyThreadState};
+use std::os::raw::c_int;
+
+// Py_tracefunc is defined in ffi::pystate
+
+pub const PyTrace_CALL: c_int = 0;
+pub const PyTrace_EXCEPTION: c_int = 1;
+pub const PyTrace_LINE: c_int = 2;
+pub const PyTrace_RETURN: c_int = 3;
+pub const PyTrace_C_CALL: c_int = 4;
+pub const PyTrace_C_EXCEPTION: c_int = 5;
+pub const PyTrace_C_RETURN: c_int = 6;
+pub const PyTrace_OPCODE: c_int = 7;
+
+extern "C" {
+    // PyGILState_Check is defined in ffi::pystate
+    pub fn PyInterpreterState_Main() -> *mut PyInterpreterState;
+    pub fn PyInterpreterState_Head() -> *mut PyInterpreterState;
+    pub fn PyInterpreterState_Next(interp: *mut PyInterpreterState) -> *mut PyInterpreterState;
+    pub fn PyInterpreterState_ThreadHead(interp: *mut PyInterpreterState) -> *mut PyThreadState;
+    pub fn PyThreadState_Next(tstate: *mut PyThreadState) -> *mut PyThreadState;
+}
+
+#[cfg(Py_3_9)]
+#[cfg_attr(docsrs, doc(cfg(Py_3_9)))]
+pub type _PyFrameEvalFunction = extern "C" fn(
+    *mut crate::ffi::PyThreadState,
+    *mut crate::ffi::PyFrameObject,
+    c_int,
+) -> *mut crate::ffi::object::PyObject;
+
+#[cfg(Py_3_9)]
+extern "C" {
+    /// Get the frame evaluation function.
+    #[cfg_attr(docsrs, doc(cfg(Py_3_9)))]
+    pub fn _PyInterpreterState_GetEvalFrameFunc(
+        interp: *mut PyInterpreterState,
+    ) -> _PyFrameEvalFunction;
+
+    ///Set the frame evaluation function.
+    #[cfg_attr(docsrs, doc(cfg(Py_3_9)))]
+    pub fn _PyInterpreterState_SetEvalFrameFunc(
+        interp: *mut PyInterpreterState,
+        eval_frame: _PyFrameEvalFunction,
+    );
+}

src/ffi/mod.rs

@@ -1,4 +1,19 @@
-//! Raw ffi declarations for the C interface of Python.
+//! Raw FFI declarations for Python's C API.
+//!
+//! This module provides low level bindings to the Python interpreter.
+//! It is meant for advanced users only - regular PyO3 users shouldn't
+//! need to interact with this module at all.
+//!
+//! # Safety
+//!
+//! The functions in this module lack individual safety documentation, but
+//! generally the following apply:
+//! - Pointer arguments have to point to a valid Python object of the correct type,
+//! although null pointers are sometimes valid input.
+//! - The vast majority can only be used safely while the GIL is held.
+//! - Some functions have additional safety requirements, consult the
+//! [Python/C API Reference Manual](https://docs.python.org/3/c-api/index.html)
+//! for more information.
 #![allow(
     non_camel_case_types,
     non_snake_case,
@@ -27,6 +42,7 @@ pub use self::code::*;
 pub use self::codecs::*;
 pub use self::compile::*;
 pub use self::complexobject::*;
+#[cfg(all(Py_3_8, not(Py_LIMITED_API)))]
 pub use self::context::*;
 #[cfg(not(Py_LIMITED_API))]
 pub use self::datetime::*;
@@ -78,6 +94,7 @@ pub use self::weakrefobject::*;
 
 #[cfg(not(Py_LIMITED_API))]
 pub use self::cpython::*;
+
 mod abstract_;
 // skipped asdl.h
 // skipped ast.h
@@ -95,8 +112,6 @@ mod compile; // TODO: incomplete
 mod complexobject; // TODO supports PEP-384 only
 #[cfg(all(Py_3_8, not(Py_LIMITED_API)))]
 mod context; // It's actually 3.7.1, but no cfg for patches.
-#[cfg(not(all(Py_3_8, not(Py_LIMITED_API))))]
-mod context {}
 #[cfg(not(Py_LIMITED_API))]
 pub(crate) mod datetime;
 mod descrobject; // TODO supports PEP-384 only

src/ffi/pystate.rs

@@ -1,17 +1,11 @@
-use crate::ffi::ceval::_PyFrameEvalFunction;
 use crate::ffi::moduleobject::PyModuleDef;
 use crate::ffi::object::PyObject;
 use crate::ffi::PyFrameObject;
 use std::os::raw::{c_int, c_long};
 
 pub const MAX_CO_EXTRA_USERS: c_int = 255;
 
-#[repr(C)]
-#[derive(Copy, Clone)]
-pub struct PyInterpreterState {
-    pub ob_base: PyObject,
-    pub eval_frame: _PyFrameEvalFunction,
-}
+opaque_struct!(PyInterpreterState);
 
 #[repr(C)]
 #[derive(Copy, Clone)]