diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fa0a212378..f0c5f0c239f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,7 +88,7 @@ jobs: id: settings shell: bash run: | - echo "::set-output name=all_additive_features::macros num-bigint num-complex hashbrown serde" + echo "::set-output name=all_additive_features::macros num-bigint num-complex hashbrown serde multiple-pymethods" - name: Build docs run: cargo doc --no-default-features --features "${{ steps.settings.outputs.all_additive_features }}" @@ -155,16 +155,15 @@ jobs: toolchain: nightly override: true profile: minimal - - uses: actions-rs/cargo@v1 - with: - command: test - args: --features "num-bigint num-complex hashbrown serde" --no-fail-fast - env: - CARGO_INCREMENTAL: 0 - RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests" - RUSTDOCFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests" + - run: cargo test --no-default-features --no-fail-fast + - run: cargo test --no-default-features --no-fail-fast --features "macros num-bigint num-complex hashbrown serde multiple-pymethods" - uses: actions-rs/grcov@v0.1 id: coverage - uses: codecov/codecov-action@v1 with: file: ${{ steps.coverage.outputs.report }} + env: + CARGO_TERM_VERBOSE: true + CARGO_INCREMENTAL: 0 + RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests" + RUSTDOCFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests" diff --git a/Cargo.toml b/Cargo.toml index 9419c71c69f..9d7ed4fb668 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,10 @@ serde_json = "1.0.61" default = ["macros"] # Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc. -macros = ["pyo3-macros", "indoc", "inventory", "paste", "unindent"] +macros = ["pyo3-macros", "indoc", "paste", "unindent"] + +# Enables multiple #[pymethods] per #[pyclass] +multiple-pymethods = ["inventory"] # Use this feature when building an extension module. # It tells the linker to keep the python symbols unresolved, diff --git a/guide/src/class.md b/guide/src/class.md index a6f3f68fc54..0ada756458d 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -1,8 +1,10 @@ # Python Classes -PyO3 exposes a group of attributes powered by Rust's proc macro system for defining Python classes as Rust structs. This chapter will discuss the functionality and configuration they offer. +PyO3 exposes a group of attributes powered by Rust's proc macro system for defining Python classes as Rust structs. -For ease of discovery, below is a list of all custom attributes with links to the relevant section of this chapter: +The main attribute is `#[pyclass]`, which is placed upon a Rust `struct` to generate a Python type for it. A struct will usually also have *one* `#[pymethods]`-annotated `impl` block for the struct, which is used to define Python methods and constants for the generated Python type. (If the [`multiple-pymethods`] feature is enabled each `#[pyclass]` is allowed to have multiple `#[pymethods]` blocks.) Finally, there may be multiple `#[pyproto]` trait implementations for the struct, which are used to define certain python magic methods such as `__str__`. + +This chapter will discuss the functionality and configuration these attributes offer. Below is a list of links to the relevant section of this chapter for each: - [`#[pyclass]`](#defining-a-new-class) - [`#[pyo3(get, set)]`](#object-properties-using-pyo3get-set) @@ -31,9 +33,9 @@ struct MyClass { } ``` -Because Python objects are freely shared between threads by the Python interpreter, all structs annotated with `#[pyclass]` must implement `Send`. +Because Python objects are freely shared between threads by the Python interpreter, all structs annotated with `#[pyclass]` must implement `Send` (unless annotated with [`#[pyclass(unsendable)]`](#customizing-the-class)). -The above example generates implementations for [`PyTypeInfo`], [`PyTypeObject`], and [`PyClass`] for `MyClass`. To see these generated implementations, refer to the section [How methods are implemented](#how-methods-are-implemented) at the end of this chapter. +The above example generates implementations for [`PyTypeInfo`], [`PyTypeObject`], and [`PyClass`] for `MyClass`. To see these generated implementations, refer to the [implementation details](#implementation-details) at the end of this chapter. ## Adding the class to a module @@ -332,7 +334,7 @@ impl SubClass { PyO3 supports two ways to add properties to your `#[pyclass]`: - For simple fields with no side effects, a `#[pyo3(get, set)]` attribute can be added directly to the field definition in the `#[pyclass]`. -- For properties which require computation you can define `#[getter]` and `#[setter]` functions in the `#[pymethods]` block. +- For properties which require computation you can define `#[getter]` and `#[setter]` functions in the [`#[pymethods]`](#instance-methods) block. We'll cover each of these in the following sections. @@ -444,7 +446,8 @@ To define a Python compatible method, an `impl` block for your struct has to be block with some variations, like descriptors, class method static methods, etc. Since Rust allows any number of `impl` blocks, you can easily split methods -between those accessible to Python (and Rust) and those accessible only to Rust. +between those accessible to Python (and Rust) and those accessible only to Rust. However to have multiple +`#[pymethods]`-annotated `impl` blocks for the same struct you must enable the [`multiple-pymethods`] feature of PyO3. ```rust # use pyo3::prelude::*; @@ -698,20 +701,21 @@ num=44, debug=false num=-1, debug=false ``` -## How methods are implemented +## Implementation details + +The `#[pyclass]` macros rely on a lot of conditional code generation: each `#[pyclass]` can optionally have a `#[pymethods]` block as well as several different possible `#[pyproto]` trait implementations. + +To support this flexibility the `#[pyclass]` macro expands to a blob of boilerplate code which sets up the structure for ["dtolnay specialization"](https://github.com/dtolnay/case-studies/blob/master/autoref-specialization/README.md). This implementation pattern enables the Rust compiler to use `#[pymethods]` and `#[pyproto]` implementations when they are present, and fall back to default (empty) definitions when they are not. -Users should be able to define a `#[pyclass]` with or without `#[pymethods]`, while PyO3 needs a -trait with a function that returns all methods. Since it's impossible to make the code generation in -pyclass dependent on whether there is an impl block, we'd need to implement the trait on -`#[pyclass]` and override the implementation in `#[pymethods]`. -To enable this, we use a static registry type provided by [inventory](https://github.com/dtolnay/inventory), -which allows us to collect `impl`s from arbitrary source code by exploiting some binary trick. -See [inventory: how it works](https://github.com/dtolnay/inventory#how-it-works) and `pyo3_macros_backend::py_class` for more details. +This simple technique works for the case when there is zero or one implementations. To support multiple `#[pymethods]` for a `#[pyclass]` (in the [`multiple-pymethods`] feature), a registry mechanism provided by the [`inventory`](https://github.com/dtolnay/inventory) crate is used instead. This collects `impl`s at library load time, but isn't supported on all platforms. See [inventory: how it works](https://github.com/dtolnay/inventory#how-it-works) for more details. -Specifically, the following implementation is generated: +The `#[pyclass]` macro expands to roughly the code seen below. The `PyClassImplCollector` is the type used internally by PyO3 for dtolnay specialization: ```rust -use pyo3::prelude::*; +# #[cfg(not(feature = "multiple-pymethods"))] +# { +# use pyo3::prelude::*; +// Note: the implementation differs slightly with the `multiple-pymethods` feature enabled. /// Class for demonstration struct MyClass { @@ -754,31 +758,14 @@ impl pyo3::IntoPy for MyClass { } } -pub struct Pyo3MethodsInventoryForMyClass { - methods: Vec, -} -impl pyo3::class::methods::PyMethodsInventory for Pyo3MethodsInventoryForMyClass { - fn new(methods: Vec) -> Self { - Self { methods } - } - fn get(&'static self) -> &'static [pyo3::class::PyMethodDefType] { - &self.methods - } -} -impl pyo3::class::methods::HasMethodsInventory for MyClass { - type Methods = Pyo3MethodsInventoryForMyClass; -} -pyo3::inventory::collect!(Pyo3MethodsInventoryForMyClass); - impl pyo3::class::impl_::PyClassImpl for MyClass { type ThreadChecker = pyo3::class::impl_::ThreadCheckerStub; fn for_each_method_def(visitor: impl FnMut(&pyo3::class::PyMethodDefType)) { use pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); - pyo3::inventory::iter::<::Methods> - .into_iter() - .flat_map(pyo3::class::methods::PyMethodsInventory::get) + collector.py_methods().iter() + .chain(collector.py_class_descriptors()) .chain(collector.object_protocol_methods()) .chain(collector.async_protocol_methods()) .chain(collector.context_protocol_methods()) @@ -824,6 +811,7 @@ impl pyo3::class::impl_::PyClassImpl for MyClass { # let py = gil.python(); # let cls = py.get_type::(); # pyo3::py_run!(py, cls, "assert cls.__name__ == 'MyClass'") +# } ``` @@ -840,3 +828,5 @@ impl pyo3::class::impl_::PyClassImpl for MyClass { [`RefCell`]: https://doc.rust-lang.org/std/cell/struct.RefCell.html [classattr]: https://docs.python.org/3/tutorial/classes.html#class-and-instance-variables + +[`multiple-pymethods`]: features.md#multiple-pymethods diff --git a/guide/src/features.md b/guide/src/features.md index 315532ad8ca..ec1ec3cc58f 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -55,6 +55,14 @@ These macros require a number of dependencies which may not be needed by users w > This feature is enabled by default. To disable it, set `default-features = false` for the `pyo3` entry in your Cargo.toml. +### `multiple-pymethods` + +This feature enables a dependency on `inventory`, which enables each `#[pyclass]` to have more than one `#[pymethods]` block. + +Most users should only need a single `#[pymethods]` per `#[pyclass]`. In addition, not all platforms (e.g. Wasm) are supported by `inventory`. For this reason this feature is not enabled by default, meaning fewer dependencies and faster compilation for the majority of users. + +See [the `#[pyclass]` implementation details](class.md#implementation-details) for more information. + ### `nightly` The `nightly` feature needs the nightly Rust compiler. This allows PyO3 to use Rust's unstable specialization feature to apply the following optimizations: diff --git a/guide/src/migration.md b/guide/src/migration.md index 9e2ae2ba99c..cafba539b22 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -9,6 +9,12 @@ For a detailed list of all changes, see the [CHANGELOG](changelog.md). For projects embedding Python in Rust, PyO3 no longer automatically initalizes a Python interpreter on the first call to `Python::with_gil` (or `Python::acquire_gil`) unless the [`auto-initalize` feature](features.md#auto-initalize) is enabled. +### New `multiple-pymethods` feature + +`#[pymethods]` have been reworked with a simpler default implementation which removes the dependency on the `inventory` crate. This reduces dependencies and compile times for the majority of users. + +The limitation of the new default implementation is that it cannot support multiple `#[pymethods]` blocks for the same `#[pyclass]`. If you need this functionality, you must enable the `multiple-pymethods` feature which will switch `#[pymethods]` to the inventory-based implementation. + ## from 0.12.* to 0.13 ### Minimum Rust version increased to Rust 1.45 diff --git a/pyo3-macros-backend/src/lib.rs b/pyo3-macros-backend/src/lib.rs index 9738e01f46a..d2198c7243e 100644 --- a/pyo3-macros-backend/src/lib.rs +++ b/pyo3-macros-backend/src/lib.rs @@ -24,6 +24,6 @@ pub use from_pyobject::build_derive_from_pyobject; pub use module::{add_fn_to_module, process_functions_in_module, py_init}; pub use pyclass::{build_py_class, PyClassArgs}; pub use pyfunction::{build_py_function, PyFunctionAttr}; -pub use pyimpl::build_py_methods; +pub use pyimpl::{build_py_methods, PyClassMethodsType}; pub use pyproto::build_py_proto; pub use utils::get_doc; diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index e46e6e39e1f..ca150948f21 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1,6 +1,7 @@ // Copyright (c) 2017-present PyO3 Project and Contributors use crate::method::{FnType, SelfType}; +use crate::pyimpl::PyClassMethodsType; use crate::pymethod::{ impl_py_getter_def, impl_py_setter_def, impl_wrap_getter, impl_wrap_setter, PropertyType, }; @@ -154,7 +155,11 @@ impl PyClassArgs { } } -pub fn build_py_class(class: &mut syn::ItemStruct, attr: &PyClassArgs) -> syn::Result { +pub fn build_py_class( + class: &mut syn::ItemStruct, + attr: &PyClassArgs, + methods_type: PyClassMethodsType, +) -> syn::Result { let text_signature = utils::parse_text_signature_attrs( &mut class.attrs, &get_class_python_name(&class.ident, attr), @@ -178,7 +183,7 @@ pub fn build_py_class(class: &mut syn::ItemStruct, attr: &PyClassArgs) -> syn::R bail_spanned!(class.fields.span() => "#[pyclass] can only be used with C-style structs"); } - impl_class(&class.ident, &attr, doc, descriptors) + impl_class(&class.ident, &attr, doc, descriptors, methods_type) } /// Parses `#[pyo3(get, set)]` @@ -210,7 +215,7 @@ fn parse_descriptors(item: &mut syn::Field) -> syn::Result> { Ok(descs) } -/// To allow multiple #[pymethods]/#[pyproto] block, we define inventory types. +/// To allow multiple #[pymethods] block, we define inventory types. fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream { // Try to build a unique type for better error messages let name = format!("Pyo3MethodsInventoryFor{}", cls.unraw()); @@ -221,7 +226,7 @@ fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream { pub struct #inventory_cls { methods: Vec, } - impl pyo3::class::methods::PyMethodsInventory for #inventory_cls { + impl pyo3::class::impl_::PyMethodsInventory for #inventory_cls { fn new(methods: Vec) -> Self { Self { methods } } @@ -230,7 +235,7 @@ fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream { } } - impl pyo3::class::methods::HasMethodsInventory for #cls { + impl pyo3::class::impl_::HasMethodsInventory for #cls { type Methods = #inventory_cls; } @@ -247,6 +252,7 @@ fn impl_class( attr: &PyClassArgs, doc: syn::LitStr, descriptors: Vec<(syn::Field, Vec)>, + methods_type: PyClassMethodsType, ) -> syn::Result { let cls_name = get_class_python_name(cls, attr).to_string(); @@ -338,7 +344,17 @@ fn impl_class( quote! {} }; - let impl_inventory = impl_methods_inventory(&cls); + let (impl_inventory, iter_py_methods) = match methods_type { + PyClassMethodsType::Specialization => (None, quote! { collector.py_methods().iter() }), + PyClassMethodsType::Inventory => ( + Some(impl_methods_inventory(&cls)), + quote! { + pyo3::inventory::iter::<::Methods> + .into_iter() + .flat_map(pyo3::class::impl_::PyMethodsInventory::get) + }, + ), + }; let base = &attr.base; let flags = &attr.flags; @@ -429,9 +445,8 @@ fn impl_class( fn for_each_method_def(visitor: impl FnMut(&pyo3::class::PyMethodDefType)) { use pyo3::class::impl_::*; let collector = PyClassImplCollector::::new(); - pyo3::inventory::iter::<::Methods> - .into_iter() - .flat_map(pyo3::class::methods::PyMethodsInventory::get) + #iter_py_methods + .chain(collector.py_class_descriptors()) .chain(collector.object_protocol_methods()) .chain(collector.async_protocol_methods()) .chain(collector.context_protocol_methods()) @@ -513,10 +528,12 @@ fn impl_descriptors( .collect::>()?; Ok(quote! { - pyo3::inventory::submit! { - #![crate = pyo3] { - type Inventory = <#cls as pyo3::class::methods::HasMethodsInventory>::Methods; - ::new(vec![#(#py_methods),*]) + impl pyo3::class::impl_::PyClassDescriptors<#cls> + for pyo3::class::impl_::PyClassImplCollector<#cls> + { + fn py_class_descriptors(self) -> &'static [pyo3::class::methods::PyMethodDefType] { + static METHODS: &[pyo3::class::methods::PyMethodDefType] = &[#(#py_methods),*]; + METHODS } } }) diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index 02cda6b6228..27ad79331cf 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -6,7 +6,16 @@ use pymethod::GeneratedPyMethod; use quote::quote; use syn::spanned::Spanned; -pub fn build_py_methods(ast: &mut syn::ItemImpl) -> syn::Result { +/// The mechanism used to collect `#[pymethods]` into the type object +pub enum PyClassMethodsType { + Specialization, + Inventory, +} + +pub fn build_py_methods( + ast: &mut syn::ItemImpl, + methods_type: PyClassMethodsType, +) -> syn::Result { if let Some((_, path, _)) = &ast.trait_ { bail_spanned!(path.span() => "#[pymethods] cannot be used on trait impl blocks"); } else if ast.generics != Default::default() { @@ -15,11 +24,15 @@ pub fn build_py_methods(ast: &mut syn::ItemImpl) -> syn::Result { "#[pymethods] cannot be used with lifetime parameters or generics" ); } else { - impl_methods(&ast.self_ty, &mut ast.items) + impl_methods(&ast.self_ty, &mut ast.items, methods_type) } } -pub fn impl_methods(ty: &syn::Type, impls: &mut Vec) -> syn::Result { +pub fn impl_methods( + ty: &syn::Type, + impls: &mut Vec, + methods_type: PyClassMethodsType, +) -> syn::Result { let mut new_impls = Vec::new(); let mut call_impls = Vec::new(); let mut methods = Vec::new(); @@ -51,18 +64,46 @@ pub fn impl_methods(ty: &syn::Type, impls: &mut Vec) -> syn::Resu } } + let methods_registration = match methods_type { + PyClassMethodsType::Specialization => impl_py_methods(ty, methods), + PyClassMethodsType::Inventory => submit_methods_inventory(ty, methods), + }; + Ok(quote! { #(#new_impls)* #(#call_impls)* + #methods_registration + }) +} + +fn impl_py_methods(ty: &syn::Type, methods: Vec) -> TokenStream { + quote! { + impl pyo3::class::impl_::PyMethods<#ty> + for pyo3::class::impl_::PyClassImplCollector<#ty> + { + fn py_methods(self) -> &'static [pyo3::class::methods::PyMethodDefType] { + static METHODS: &[pyo3::class::methods::PyMethodDefType] = &[#(#methods),*]; + METHODS + } + } + } +} + +fn submit_methods_inventory(ty: &syn::Type, methods: Vec) -> TokenStream { + if methods.is_empty() { + return TokenStream::default(); + } + + quote! { pyo3::inventory::submit! { #![crate = pyo3] { - type Inventory = <#ty as pyo3::class::methods::HasMethodsInventory>::Methods; - ::new(vec![#(#methods),*]) + type Inventory = <#ty as pyo3::class::impl_::HasMethodsInventory>::Methods; + ::new(vec![#(#methods),*]) } } - }) + } } fn get_cfg_attributes(attrs: &[syn::Attribute]) -> Vec<&syn::Attribute> { diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 345b679e446..09e872eed9d 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -689,7 +689,10 @@ pub fn impl_py_method_class_attribute(spec: &FnSpec<'_>, wrapper: &TokenStream) pyo3::class::PyMethodDefType::ClassAttribute({ #wrapper - pyo3::class::PyClassAttributeDef::new(concat!(stringify!(#python_name), "\0"), __wrap) + pyo3::class::PyClassAttributeDef::new( + concat!(stringify!(#python_name), "\0"), + pyo3::class::methods::PyClassAttributeFactory(__wrap) + ) }) } } @@ -700,7 +703,10 @@ pub fn impl_py_const_class_attribute(spec: &ConstSpec, wrapper: &TokenStream) -> pyo3::class::PyMethodDefType::ClassAttribute({ #wrapper - pyo3::class::PyClassAttributeDef::new(concat!(stringify!(#python_name), "\0"), __wrap) + pyo3::class::PyClassAttributeDef::new( + concat!(stringify!(#python_name), "\0"), + pyo3::class::methods::PyClassAttributeFactory(__wrap) + ) }) } } @@ -726,7 +732,11 @@ pub(crate) fn impl_py_setter_def( pyo3::class::PyMethodDefType::Setter({ #wrapper - pyo3::class::PySetterDef::new(concat!(stringify!(#python_name), "\0"), __wrap, #doc) + pyo3::class::PySetterDef::new( + concat!(stringify!(#python_name), "\0"), + pyo3::class::methods::PySetter(__wrap), + #doc + ) }) } } @@ -740,7 +750,11 @@ pub(crate) fn impl_py_getter_def( pyo3::class::PyMethodDefType::Getter({ #wrapper - pyo3::class::PyGetterDef::new(concat!(stringify!(#python_name), "\0"), __wrap, #doc) + pyo3::class::PyGetterDef::new( + concat!(stringify!(#python_name), "\0"), + pyo3::class::methods::PyGetter(__wrap), + #doc + ) }) } } diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index a4143229972..756bb5d8243 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -7,7 +7,8 @@ extern crate proc_macro; use proc_macro::TokenStream; use pyo3_macros_backend::{ build_derive_from_pyobject, build_py_class, build_py_function, build_py_methods, - build_py_proto, get_doc, process_functions_in_module, py_init, PyClassArgs, PyFunctionAttr, + build_py_proto, get_doc, process_functions_in_module, py_init, PyClassArgs, PyClassMethodsType, + PyFunctionAttr, }; use quote::quote; use syn::parse_macro_input; @@ -56,27 +57,22 @@ pub fn pyproto(_: TokenStream, input: TokenStream) -> TokenStream { #[proc_macro_attribute] pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream { - let mut ast = parse_macro_input!(input as syn::ItemStruct); - let args = parse_macro_input!(attr as PyClassArgs); - let expanded = build_py_class(&mut ast, &args).unwrap_or_else(|e| e.to_compile_error()); + pyclass_impl(attr, input, PyClassMethodsType::Specialization) +} - quote!( - #ast - #expanded - ) - .into() +#[proc_macro_attribute] +pub fn pyclass_with_inventory(attr: TokenStream, input: TokenStream) -> TokenStream { + pyclass_impl(attr, input, PyClassMethodsType::Inventory) } #[proc_macro_attribute] pub fn pymethods(_: TokenStream, input: TokenStream) -> TokenStream { - let mut ast = parse_macro_input!(input as syn::ItemImpl); - let expanded = build_py_methods(&mut ast).unwrap_or_else(|e| e.to_compile_error()); + pymethods_impl(input, PyClassMethodsType::Specialization) +} - quote!( - #ast - #expanded - ) - .into() +#[proc_macro_attribute] +pub fn pymethods_with_inventory(_: TokenStream, input: TokenStream) -> TokenStream { + pymethods_impl(input, PyClassMethodsType::Inventory) } #[proc_macro_attribute] @@ -102,3 +98,32 @@ pub fn derive_from_py_object(item: TokenStream) -> TokenStream { ) .into() } + +fn pyclass_impl( + attr: TokenStream, + input: TokenStream, + methods_type: PyClassMethodsType, +) -> TokenStream { + let mut ast = parse_macro_input!(input as syn::ItemStruct); + let args = parse_macro_input!(attr as PyClassArgs); + let expanded = + build_py_class(&mut ast, &args, methods_type).unwrap_or_else(|e| e.to_compile_error()); + + quote!( + #ast + #expanded + ) + .into() +} + +fn pymethods_impl(input: TokenStream, methods_type: PyClassMethodsType) -> TokenStream { + let mut ast = parse_macro_input!(input as syn::ItemImpl); + let expanded = + build_py_methods(&mut ast, methods_type).unwrap_or_else(|e| e.to_compile_error()); + + quote!( + #ast + #expanded + ) + .into() +} diff --git a/src/class/impl_.rs b/src/class/impl_.rs index 41bfd16f462..ae9d499a155 100644 --- a/src/class/impl_.rs +++ b/src/class/impl_.rs @@ -76,6 +76,50 @@ impl PyClassCallImpl for &'_ PyClassImplCollector { } } +// General methods implementation: either dtolnay specialization trait or inventory if +// multiple-pymethods feature is enabled. + +macro_rules! methods_trait { + ($name:ident, $function_name: ident) => { + pub trait $name { + fn $function_name(self) -> &'static [PyMethodDefType]; + } + + impl $name for &'_ PyClassImplCollector { + fn $function_name(self) -> &'static [PyMethodDefType] { + &[] + } + } + }; +} + +/// Implementation detail. Only to be used through our proc macro code. +/// Method storage for `#[pyclass]`. +/// Allows arbitrary `#[pymethod]` blocks to submit their methods, +/// which are eventually collected by `#[pyclass]`. +#[cfg(all(feature = "macros", feature = "multiple-pymethods"))] +pub trait PyMethodsInventory: inventory::Collect { + /// Create a new instance + fn new(methods: Vec) -> Self; + + /// Returns the methods for a single `#[pymethods] impl` block + fn get(&'static self) -> &'static [PyMethodDefType]; +} + +/// Implemented for `#[pyclass]` in our proc macro code. +/// Indicates that the pyclass has its own method storage. +#[cfg(all(feature = "macros", feature = "multiple-pymethods"))] +pub trait HasMethodsInventory { + type Methods: PyMethodsInventory; +} + +// Methods from #[pyo3(get, set)] on struct fields. +methods_trait!(PyClassDescriptors, py_class_descriptors); + +// Methods from #[pymethods] if not using inventory. +#[cfg(not(feature = "multiple-pymethods"))] +methods_trait!(PyMethods, py_methods); + // All traits describing slots, as well as the fallback implementations for unimplemented protos // // Protos which are implemented use dtolnay specialization to implement for PyClassImplCollector. @@ -106,20 +150,6 @@ slots_trait!(PyAsyncProtocolSlots, async_protocol_slots); slots_trait!(PySequenceProtocolSlots, sequence_protocol_slots); slots_trait!(PyBufferProtocolSlots, buffer_protocol_slots); -macro_rules! methods_trait { - ($name:ident, $function_name: ident) => { - pub trait $name { - fn $function_name(self) -> &'static [PyMethodDefType]; - } - - impl $name for &'_ PyClassImplCollector { - fn $function_name(self) -> &'static [PyMethodDefType] { - &[] - } - } - }; -} - methods_trait!(PyObjectProtocolMethods, object_protocol_methods); methods_trait!(PyAsyncProtocolMethods, async_protocol_methods); methods_trait!(PyContextProtocolMethods, context_protocol_methods); diff --git a/src/class/methods.rs b/src/class/methods.rs index df0d5a012c6..e535ed0df10 100644 --- a/src/class/methods.rs +++ b/src/class/methods.rs @@ -1,7 +1,8 @@ // Copyright (c) 2017-present PyO3 Project and Contributors +use crate::internal_tricks::{extract_cstr_or_leak_cstring, NulByteInString}; use crate::{ffi, PyObject, Python}; -use std::ffi::{CStr, CString}; +use std::ffi::CStr; use std::fmt; use std::os::raw::c_int; @@ -29,16 +30,22 @@ pub enum PyMethodType { PyCFunctionWithKeywords(PyCFunctionWithKeywords), } -// These two newtype structs serve no purpose other than wrapping the raw ffi types (which are -// function pointers) - because function pointers aren't allowed in const fn, but types wrapping -// them are! +// These newtype structs serve no purpose other than wrapping which are function pointers - because +// function pointers aren't allowed in const fn, but types wrapping them are! #[derive(Clone, Copy, Debug)] pub struct PyCFunction(pub ffi::PyCFunction); #[derive(Clone, Copy, Debug)] pub struct PyCFunctionWithKeywords(pub ffi::PyCFunctionWithKeywords); +#[derive(Clone, Copy, Debug)] +pub struct PyGetter(pub ffi::getter); +#[derive(Clone, Copy, Debug)] +pub struct PySetter(pub ffi::setter); +#[derive(Clone, Copy)] +pub struct PyClassAttributeFactory(pub for<'p> fn(Python<'p>) -> PyObject); // TODO: it would be nice to use CStr in these types, but then the constructors can't be const fn // until `CStr::from_bytes_with_nul_unchecked` is const fn. + #[derive(Clone, Debug)] pub struct PyMethodDef { pub(crate) ml_name: &'static str, @@ -49,22 +56,22 @@ pub struct PyMethodDef { #[derive(Copy, Clone)] pub struct PyClassAttributeDef { - pub(crate) name: &'static CStr, - pub(crate) meth: for<'p> fn(Python<'p>) -> PyObject, + pub(crate) name: &'static str, + pub(crate) meth: PyClassAttributeFactory, } #[derive(Clone, Debug)] pub struct PyGetterDef { - pub(crate) name: &'static CStr, - pub(crate) meth: ffi::getter, - doc: &'static CStr, + pub(crate) name: &'static str, + pub(crate) meth: PyGetter, + doc: &'static str, } #[derive(Clone, Debug)] pub struct PySetterDef { - pub(crate) name: &'static CStr, - pub(crate) meth: ffi::setter, - doc: &'static CStr, + pub(crate) name: &'static str, + pub(crate) meth: PySetter, + doc: &'static str, } unsafe impl Sync for PyMethodDef {} @@ -117,11 +124,8 @@ impl PyMethodDef { impl PyClassAttributeDef { /// Define a class attribute. - pub fn new(name: &'static str, meth: for<'p> fn(Python<'p>) -> PyObject) -> Self { - Self { - name: get_name(name).unwrap(), - meth, - } + pub const fn new(name: &'static str, meth: PyClassAttributeFactory) -> Self { + Self { name, meth } } } @@ -137,73 +141,48 @@ impl fmt::Debug for PyClassAttributeDef { impl PyGetterDef { /// Define a getter. - pub fn new(name: &'static str, getter: ffi::getter, doc: &'static str) -> Self { + pub const fn new(name: &'static str, getter: PyGetter, doc: &'static str) -> Self { Self { - name: get_name(name).unwrap(), + name, meth: getter, - doc: get_doc(doc).unwrap(), + doc, } } /// Copy descriptor information to `ffi::PyGetSetDef` pub fn copy_to(&self, dst: &mut ffi::PyGetSetDef) { if dst.name.is_null() { - dst.name = self.name.as_ptr() as _; + dst.name = get_name(self.name).unwrap().as_ptr() as _; } if dst.doc.is_null() { - dst.doc = self.doc.as_ptr() as _; + dst.doc = get_doc(self.doc).unwrap().as_ptr() as _; } - dst.get = Some(self.meth); + dst.get = Some(self.meth.0); } } impl PySetterDef { /// Define a setter. - pub fn new(name: &'static str, setter: ffi::setter, doc: &'static str) -> Self { + pub const fn new(name: &'static str, setter: PySetter, doc: &'static str) -> Self { Self { - name: get_name(name).unwrap(), + name, meth: setter, - doc: get_doc(doc).unwrap(), + doc, } } /// Copy descriptor information to `ffi::PyGetSetDef` pub fn copy_to(&self, dst: &mut ffi::PyGetSetDef) { if dst.name.is_null() { - dst.name = self.name.as_ptr() as _; + dst.name = get_name(self.name).unwrap().as_ptr() as _; } if dst.doc.is_null() { - dst.doc = self.doc.as_ptr() as _; + dst.doc = get_doc(self.doc).unwrap().as_ptr() as _; } - dst.set = Some(self.meth); + dst.set = Some(self.meth.0); } } -/// Implementation detail. Only to be used through our proc macro code. -/// Method storage for `#[pyclass]`. -/// Allows arbitrary `#[pymethod]/#[pyproto]` blocks to submit their methods, -/// which are eventually collected by `#[pyclass]`. -#[doc(hidden)] -#[cfg(feature = "macros")] -pub trait PyMethodsInventory: inventory::Collect { - /// Create a new instance - fn new(methods: Vec) -> Self; - - /// Returns the methods for a single `#[pymethods] impl` block - fn get(&'static self) -> &'static [PyMethodDefType]; -} - -/// Implemented for `#[pyclass]` in our proc macro code. -/// Indicates that the pyclass has its own method storage. -#[doc(hidden)] -#[cfg(feature = "macros")] -pub trait HasMethodsInventory { - type Methods: PyMethodsInventory; -} - -#[derive(Debug)] -pub(crate) struct NulByteInString(pub(crate) &'static str); - fn get_name(name: &'static str) -> Result<&'static CStr, NulByteInString> { extract_cstr_or_leak_cstring(name, "Function name cannot contain NUL byte.") } @@ -211,14 +190,3 @@ fn get_name(name: &'static str) -> Result<&'static CStr, NulByteInString> { fn get_doc(doc: &'static str) -> Result<&'static CStr, NulByteInString> { extract_cstr_or_leak_cstring(doc, "Document cannot contain NUL byte.") } - -fn extract_cstr_or_leak_cstring( - src: &'static str, - err_msg: &'static str, -) -> Result<&'static CStr, NulByteInString> { - CStr::from_bytes_with_nul(src.as_bytes()) - .or_else(|_| { - CString::new(src.as_bytes()).map(|c_string| &*Box::leak(c_string.into_boxed_c_str())) - }) - .map_err(|_| NulByteInString(err_msg)) -} diff --git a/src/internal_tricks.rs b/src/internal_tricks.rs index d13fed50d08..33f9bb5c4a4 100644 --- a/src/internal_tricks.rs +++ b/src/internal_tricks.rs @@ -1,3 +1,4 @@ +use std::ffi::{CStr, CString}; use std::marker::PhantomData; use std::rc::Rc; @@ -36,3 +37,17 @@ macro_rules! pyo3_exception { $crate::create_exception_type_object!(pyo3_runtime, $name, $base); }; } + +#[derive(Debug)] +pub(crate) struct NulByteInString(pub(crate) &'static str); + +pub(crate) fn extract_cstr_or_leak_cstring( + src: &'static str, + err_msg: &'static str, +) -> Result<&'static CStr, NulByteInString> { + CStr::from_bytes_with_nul(src.as_bytes()) + .or_else(|_| { + CString::new(src.as_bytes()).map(|c_string| &*Box::leak(c_string.into_boxed_c_str())) + }) + .map_err(|_| NulByteInString(err_msg)) +} diff --git a/src/lib.rs b/src/lib.rs index f11ad92769c..b2d8e752256 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -166,12 +166,14 @@ pub use crate::types::PyAny; #[cfg(feature = "macros")] #[doc(hidden)] pub use { - indoc, // Re-exported for py_run - inventory, // Re-exported for pymethods - paste, // Re-exported for wrap_function - unindent, // Re-exported for py_run + indoc, // Re-exported for py_run + paste, // Re-exported for wrap_function + unindent, // Re-exported for py_run }; +#[cfg(all(feature = "macros", feature = "multiple-pymethods"))] +pub use inventory; // Re-exported for `#[pyclass]` and `#[pymethods]` with `multiple-pymethods`. + #[macro_use] mod internal_tricks; @@ -216,7 +218,15 @@ pub mod serde; pub mod proc_macro { pub use pyo3_macros::pymodule; /// The proc macro attributes - pub use pyo3_macros::{pyclass, pyfunction, pymethods, pyproto}; + pub use pyo3_macros::{pyfunction, pyproto}; + + #[cfg(not(feature = "multiple-pymethods"))] + pub use pyo3_macros::{pyclass, pymethods}; + + #[cfg(feature = "multiple-pymethods")] + pub use pyo3_macros::{ + pyclass_with_inventory as pyclass, pymethods_with_inventory as pymethods, + }; } /// Returns a function that takes a [Python] instance and returns a Python function. diff --git a/src/prelude.rs b/src/prelude.rs index 071eb901ef9..15dbcc7c072 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -20,4 +20,4 @@ pub use crate::{FromPyObject, IntoPy, IntoPyPointer, PyTryFrom, PyTryInto, ToPyO // PyModule is only part of the prelude because we need it for the pymodule function pub use crate::types::{PyAny, PyModule}; #[cfg(feature = "macros")] -pub use pyo3_macros::{pyclass, pyfunction, pymethods, pymodule, pyproto, FromPyObject}; +pub use {crate::proc_macro::*, pyo3_macros::FromPyObject}; diff --git a/src/type_object.rs b/src/type_object.rs index 673c9938288..4db1aba8ef3 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -1,6 +1,7 @@ // Copyright (c) 2017-present PyO3 Project and Contributors //! Python type object information +use crate::internal_tricks::extract_cstr_or_leak_cstring; use crate::once_cell::GILOnceCell; use crate::pyclass::{create_type_object, PyClass}; use crate::pyclass_init::PyObjectInit; @@ -197,7 +198,14 @@ impl LazyStaticType { let mut items = vec![]; T::for_each_method_def(|def| { if let PyMethodDefType::ClassAttribute(attr) = def { - items.push((attr.name, (attr.meth)(py))); + items.push(( + extract_cstr_or_leak_cstring( + attr.name, + "class attribute name cannot contain nul bytes", + ) + .unwrap(), + (attr.meth.0)(py), + )); } }); diff --git a/src/types/function.rs b/src/types/function.rs index 6ea1cb1cc27..7303b2bc720 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -1,10 +1,10 @@ +use crate::derive_utils::PyFunctionArguments; use crate::exceptions::PyValueError; use crate::prelude::*; use crate::{ class::methods::{self, PyMethodDef}, ffi, AsPyPointer, }; -use crate::{derive_utils::PyFunctionArguments, methods::NulByteInString}; /// Represents a builtin Python function object. #[repr(transparent)] @@ -54,7 +54,7 @@ impl PyCFunction { let (py, module) = py_or_module.into_py_and_maybe_module(); let def = method_def .as_method_def() - .map_err(|NulByteInString(err)| PyValueError::new_err(err))?; + .map_err(|err| PyValueError::new_err(err.0))?; let (mod_ptr, module_name) = if let Some(m) = module { let mod_ptr = m.as_ptr(); let name = m.name()?.into_py(py); diff --git a/tests/test_multiple_pymethods.rs b/tests/test_multiple_pymethods.rs new file mode 100644 index 00000000000..ab249ea9eba --- /dev/null +++ b/tests/test_multiple_pymethods.rs @@ -0,0 +1,77 @@ +#![cfg(feature = "multiple-pymethods")] + +use pyo3::prelude::*; +use pyo3::type_object::PyTypeObject; +use pyo3::types::PyType; + +#[macro_use] +mod common; + +#[pyclass] +struct PyClassWithMultiplePyMethods {} + +#[pymethods] +impl PyClassWithMultiplePyMethods { + #[new] + fn new() -> Self { + Self {} + } +} + +#[pymethods] +impl PyClassWithMultiplePyMethods { + #[call] + fn call(&self) -> &'static str { + "call" + } +} + +#[pymethods] +impl PyClassWithMultiplePyMethods { + fn method(&self) -> &'static str { + "method" + } +} + +#[pymethods] +impl PyClassWithMultiplePyMethods { + #[classmethod] + fn classmethod(_ty: &PyType) -> &'static str { + "classmethod" + } +} + +#[pymethods] +impl PyClassWithMultiplePyMethods { + #[staticmethod] + fn staticmethod() -> &'static str { + "staticmethod" + } +} + +#[pymethods] +impl PyClassWithMultiplePyMethods { + #[classattr] + fn class_attribute() -> &'static str { + "class_attribute" + } +} + +#[pymethods] +impl PyClassWithMultiplePyMethods { + #[classattr] + const CLASS_ATTRIBUTE: &'static str = "CLASS_ATTRIBUTE"; +} + +#[test] +fn test_class_with_multiple_pymethods() { + Python::with_gil(|py| { + let cls = PyClassWithMultiplePyMethods::type_object(py); + py_assert!(py, cls, "cls()() == 'call'"); + py_assert!(py, cls, "cls().method() == 'method'"); + py_assert!(py, cls, "cls.classmethod() == 'classmethod'"); + py_assert!(py, cls, "cls.staticmethod() == 'staticmethod'"); + py_assert!(py, cls, "cls.class_attribute == 'class_attribute'"); + py_assert!(py, cls, "cls.CLASS_ATTRIBUTE == 'CLASS_ATTRIBUTE'"); + }) +}