Merge pull request #1457 from davidhewitt/multiple-pymethods

pymethods: make inventory optional

.github/workflows/ci.yml

@@ -88,7 +88,7 @@ jobs:
         id: settings
         shell: bash
         run: |
-          echo "::set-output name=all_additive_features::macros num-bigint num-complex hashbrown serde"
+          echo "::set-output name=all_additive_features::macros num-bigint num-complex hashbrown serde multiple-pymethods"
 
       - name: Build docs
         run: cargo doc --no-default-features --features "${{ steps.settings.outputs.all_additive_features }}"
@@ -156,16 +156,15 @@ jobs:
           toolchain: nightly
           override: true
           profile: minimal
-      - uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --features "num-bigint num-complex hashbrown serde" --no-fail-fast
-        env:
-          CARGO_INCREMENTAL: 0
-          RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"
-          RUSTDOCFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"
+      - run: cargo test --no-default-features --no-fail-fast
+      - run: cargo test --no-default-features --no-fail-fast --features "macros num-bigint num-complex hashbrown serde multiple-pymethods"
       - uses: actions-rs/grcov@v0.1
         id: coverage
       - uses: codecov/codecov-action@v1
         with:
           file: ${{ steps.coverage.outputs.report }}
+    env:
+      CARGO_TERM_VERBOSE: true
+      CARGO_INCREMENTAL: 0
+      RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"
+      RUSTDOCFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"

Cargo.toml

@@ -43,7 +43,10 @@ serde_json = "1.0.61"
 default = ["macros"]
 
 # Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc.
-macros = ["pyo3-macros", "indoc", "inventory", "paste", "unindent"]
+macros = ["pyo3-macros", "indoc", "paste", "unindent"]
+
+# Enables multiple #[pymethods] per #[pyclass]
+multiple-pymethods = ["inventory"]
 
 # Use this feature when building an extension module.
 # It tells the linker to keep the python symbols unresolved,

guide/src/class.md

@@ -1,8 +1,10 @@
 # Python Classes
 
-PyO3 exposes a group of attributes powered by Rust's proc macro system for defining Python classes as Rust structs. This chapter will discuss the functionality and configuration they offer.
+PyO3 exposes a group of attributes powered by Rust's proc macro system for defining Python classes as Rust structs.
 
-For ease of discovery, below is a list of all custom attributes with links to the relevant section of this chapter:
+The main attribute is `#[pyclass]`, which is placed upon a Rust `struct` to generate a Python type for it. A struct will usually also have *one* `#[pymethods]`-annotated `impl` block for the struct, which is used to define Python methods and constants for the generated Python type. (If the [`multiple-pymethods`] feature is enabled each `#[pyclass]` is allowed to have multiple `#[pymethods]` blocks.) Finally, there may be multiple `#[pyproto]` trait implementations for the struct, which are used to define certain python magic methods such as `__str__`.
+
+This chapter will discuss the functionality and configuration these attributes offer. Below is a list of links to the relevant section of this chapter for each:
 
 - [`#[pyclass]`](#defining-a-new-class)
   - [`#[pyo3(get, set)]`](#object-properties-using-pyo3get-set)
@@ -31,9 +33,9 @@ struct MyClass {
 }
 ```
 
-Because Python objects are freely shared between threads by the Python interpreter, all structs annotated with `#[pyclass]` must implement `Send`.
+Because Python objects are freely shared between threads by the Python interpreter, all structs annotated with `#[pyclass]` must implement `Send` (unless annotated with [`#[pyclass(unsendable)]`](#customizing-the-class)).
 
-The above example generates implementations for [`PyTypeInfo`], [`PyTypeObject`], and [`PyClass`] for `MyClass`. To see these generated implementations, refer to the section [How methods are implemented](#how-methods-are-implemented) at the end of this chapter.
+The above example generates implementations for [`PyTypeInfo`], [`PyTypeObject`], and [`PyClass`] for `MyClass`. To see these generated implementations, refer to the [implementation details](#implementation-details) at the end of this chapter.
 
 ## Adding the class to a module
 
@@ -332,7 +334,7 @@ impl SubClass {
 
 PyO3 supports two ways to add properties to your `#[pyclass]`:
 - For simple fields with no side effects, a `#[pyo3(get, set)]` attribute can be added directly to the field definition in the `#[pyclass]`.
-- For properties which require computation you can define `#[getter]` and `#[setter]` functions in the `#[pymethods]` block.
+- For properties which require computation you can define `#[getter]` and `#[setter]` functions in the [`#[pymethods]`](#instance-methods) block.
 
 We'll cover each of these in the following sections.
 
@@ -444,7 +446,8 @@ To define a Python compatible method, an `impl` block for your struct has to be
 block with some variations, like descriptors, class method static methods, etc.
 
 Since Rust allows any number of `impl` blocks, you can easily split methods
-between those accessible to Python (and Rust) and those accessible only to Rust.
+between those accessible to Python (and Rust) and those accessible only to Rust. However to have multiple
+`#[pymethods]`-annotated `impl` blocks for the same struct you must enable the [`multiple-pymethods`] feature of PyO3.
 
 ```rust
 # use pyo3::prelude::*;
@@ -698,20 +701,21 @@ num=44, debug=false
 num=-1, debug=false
 ```
 
-## How methods are implemented
+## Implementation details
+
+The `#[pyclass]` macros rely on a lot of conditional code generation: each `#[pyclass]` can optionally have a `#[pymethods]` block as well as several different possible `#[pyproto]` trait implementations.
+
+To support this flexibility the `#[pyclass]` macro expands to a blob of boilerplate code which sets up the structure for ["dtolnay specialization"](https://github.com/dtolnay/case-studies/blob/master/autoref-specialization/README.md). This implementation pattern enables the Rust compiler to use `#[pymethods]` and `#[pyproto]` implementations when they are present, and fall back to default (empty) definitions when they are not.
 
-Users should be able to define a `#[pyclass]` with or without `#[pymethods]`, while PyO3 needs a
-trait with a function that returns all methods. Since it's impossible to make the code generation in
-pyclass dependent on whether there is an impl block, we'd need to implement the trait on
-`#[pyclass]` and override the implementation in `#[pymethods]`.
-To enable this, we use a static registry type provided by [inventory](https://github.com/dtolnay/inventory),
-which allows us to collect `impl`s from arbitrary source code by exploiting some binary trick.
-See [inventory: how it works](https://github.com/dtolnay/inventory#how-it-works) and `pyo3_macros_backend::py_class` for more details.
+This simple technique works for the case when there is zero or one implementations. To support multiple `#[pymethods]` for a `#[pyclass]` (in the [`multiple-pymethods`] feature), a registry mechanism provided by the [`inventory`](https://github.com/dtolnay/inventory) crate is used instead. This collects `impl`s at library load time, but isn't supported on all platforms. See [inventory: how it works](https://github.com/dtolnay/inventory#how-it-works) for more details.
 
-Specifically, the following implementation is generated:
+The `#[pyclass]` macro expands to roughly the code seen below. The `PyClassImplCollector` is the type used internally by PyO3 for dtolnay specialization:
 
 ```rust
-use pyo3::prelude::*;
+# #[cfg(not(feature = "multiple-pymethods"))]
+# {
+# use pyo3::prelude::*;
+// Note: the implementation differs slightly with the `multiple-pymethods` feature enabled.
 
 /// Class for demonstration
 struct MyClass {
@@ -754,31 +758,14 @@ impl pyo3::IntoPy<PyObject> for MyClass {
     }
 }
 
-pub struct Pyo3MethodsInventoryForMyClass {
-    methods: Vec<pyo3::class::PyMethodDefType>,
-}
-impl pyo3::class::methods::PyMethodsInventory for Pyo3MethodsInventoryForMyClass {
-    fn new(methods: Vec<pyo3::class::PyMethodDefType>) -> Self {
-        Self { methods }
-    }
-    fn get(&'static self) -> &'static [pyo3::class::PyMethodDefType] {
-        &self.methods
-    }
-}
-impl pyo3::class::methods::HasMethodsInventory for MyClass {
-    type Methods = Pyo3MethodsInventoryForMyClass;
-}
-pyo3::inventory::collect!(Pyo3MethodsInventoryForMyClass);
-
 impl pyo3::class::impl_::PyClassImpl for MyClass {
     type ThreadChecker = pyo3::class::impl_::ThreadCheckerStub<MyClass>;
 
     fn for_each_method_def(visitor: impl FnMut(&pyo3::class::PyMethodDefType)) {
         use pyo3::class::impl_::*;
         let collector = PyClassImplCollector::<MyClass>::new();
-        pyo3::inventory::iter::<<MyClass as pyo3::class::methods::HasMethodsInventory>::Methods>
-            .into_iter()
-            .flat_map(pyo3::class::methods::PyMethodsInventory::get)
+        collector.py_methods().iter()
+            .chain(collector.py_class_descriptors())
             .chain(collector.object_protocol_methods())
             .chain(collector.async_protocol_methods())
             .chain(collector.context_protocol_methods())
@@ -824,6 +811,7 @@ impl pyo3::class::impl_::PyClassImpl for MyClass {
 # let py = gil.python();
 # let cls = py.get_type::<MyClass>();
 # pyo3::py_run!(py, cls, "assert cls.__name__ == 'MyClass'")
+# }
 ```
 
 
@@ -840,3 +828,5 @@ impl pyo3::class::impl_::PyClassImpl for MyClass {
 [`RefCell`]: https://doc.rust-lang.org/std/cell/struct.RefCell.html
 
 [classattr]: https://docs.python.org/3/tutorial/classes.html#class-and-instance-variables
+
+[`multiple-pymethods`]: features.md#multiple-pymethods

guide/src/features.md

@@ -55,6 +55,14 @@ These macros require a number of dependencies which may not be needed by users w
 
 > This feature is enabled by default. To disable it, set `default-features = false` for the `pyo3` entry in your Cargo.toml.
 
+### `multiple-pymethods`
+
+This feature enables a dependency on `inventory`, which enables each `#[pyclass]` to have more than one `#[pymethods]` block.
+
+Most users should only need a single `#[pymethods]` per `#[pyclass]`. In addition, not all platforms (e.g. Wasm) are supported by `inventory`. For this reason this feature is not enabled by default, meaning fewer dependencies and faster compilation for the majority of users.
+
+See [the `#[pyclass]` implementation details](class.md#implementation-details) for more information.
+
 ### `nightly`
 
 The `nightly` feature needs the nightly Rust compiler. This allows PyO3 to use Rust's unstable specialization feature to apply the following optimizations:

guide/src/migration.md

@@ -9,6 +9,12 @@ For a detailed list of all changes, see the [CHANGELOG](changelog.md).
 
 For projects embedding Python in Rust, PyO3 no longer automatically initalizes a Python interpreter on the first call to `Python::with_gil` (or `Python::acquire_gil`) unless the [`auto-initalize` feature](features.md#auto-initalize) is enabled.
 
+### New `multiple-pymethods` feature
+
+`#[pymethods]` have been reworked with a simpler default implementation which removes the dependency on the `inventory` crate. This reduces dependencies and compile times for the majority of users.
+
+The limitation of the new default implementation is that it cannot support multiple `#[pymethods]` blocks for the same `#[pyclass]`. If you need this functionality, you must enable the `multiple-pymethods` feature which will switch `#[pymethods]` to the inventory-based implementation.
+
 ## from 0.12.* to 0.13
 
 ### Minimum Rust version increased to Rust 1.45

pyo3-macros-backend/src/lib.rs

@@ -24,6 +24,6 @@ pub use from_pyobject::build_derive_from_pyobject;
 pub use module::{add_fn_to_module, process_functions_in_module, py_init};
 pub use pyclass::{build_py_class, PyClassArgs};
 pub use pyfunction::{build_py_function, PyFunctionAttr};
-pub use pyimpl::build_py_methods;
+pub use pyimpl::{build_py_methods, PyClassMethodsType};
 pub use pyproto::build_py_proto;
 pub use utils::get_doc;

pyo3-macros-backend/src/pyclass.rs

@@ -1,6 +1,7 @@
 // Copyright (c) 2017-present PyO3 Project and Contributors
 
 use crate::method::{FnType, SelfType};
+use crate::pyimpl::PyClassMethodsType;
 use crate::pymethod::{
     impl_py_getter_def, impl_py_setter_def, impl_wrap_getter, impl_wrap_setter, PropertyType,
 };
@@ -154,7 +155,11 @@ impl PyClassArgs {
     }
 }
 
-pub fn build_py_class(class: &mut syn::ItemStruct, attr: &PyClassArgs) -> syn::Result<TokenStream> {
+pub fn build_py_class(
+    class: &mut syn::ItemStruct,
+    attr: &PyClassArgs,
+    methods_type: PyClassMethodsType,
+) -> syn::Result<TokenStream> {
     let text_signature = utils::parse_text_signature_attrs(
         &mut class.attrs,
         &get_class_python_name(&class.ident, attr),
@@ -178,7 +183,7 @@ pub fn build_py_class(class: &mut syn::ItemStruct, attr: &PyClassArgs) -> syn::R
         bail_spanned!(class.fields.span() => "#[pyclass] can only be used with C-style structs");
     }
 
-    impl_class(&class.ident, &attr, doc, descriptors)
+    impl_class(&class.ident, &attr, doc, descriptors, methods_type)
 }
 
 /// Parses `#[pyo3(get, set)]`
@@ -210,7 +215,7 @@ fn parse_descriptors(item: &mut syn::Field) -> syn::Result<Vec<FnType>> {
     Ok(descs)
 }
 
-/// To allow multiple #[pymethods]/#[pyproto] block, we define inventory types.
+/// To allow multiple #[pymethods] block, we define inventory types.
 fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream {
     // Try to build a unique type for better error messages
     let name = format!("Pyo3MethodsInventoryFor{}", cls.unraw());
@@ -221,7 +226,7 @@ fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream {
         pub struct #inventory_cls {
             methods: Vec<pyo3::class::PyMethodDefType>,
         }
-        impl pyo3::class::methods::PyMethodsInventory for #inventory_cls {
+        impl pyo3::class::impl_::PyMethodsInventory for #inventory_cls {
             fn new(methods: Vec<pyo3::class::PyMethodDefType>) -> Self {
                 Self { methods }
             }
@@ -230,7 +235,7 @@ fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream {
             }
         }
 
-        impl pyo3::class::methods::HasMethodsInventory for #cls {
+        impl pyo3::class::impl_::HasMethodsInventory for #cls {
             type Methods = #inventory_cls;
         }
 
@@ -247,6 +252,7 @@ fn impl_class(
     attr: &PyClassArgs,
     doc: syn::LitStr,
     descriptors: Vec<(syn::Field, Vec<FnType>)>,
+    methods_type: PyClassMethodsType,
 ) -> syn::Result<TokenStream> {
     let cls_name = get_class_python_name(cls, attr).to_string();
 
@@ -338,7 +344,17 @@ fn impl_class(
         quote! {}
     };
 
-    let impl_inventory = impl_methods_inventory(&cls);
+    let (impl_inventory, iter_py_methods) = match methods_type {
+        PyClassMethodsType::Specialization => (None, quote! { collector.py_methods().iter() }),
+        PyClassMethodsType::Inventory => (
+            Some(impl_methods_inventory(&cls)),
+            quote! {
+                pyo3::inventory::iter::<<Self as pyo3::class::impl_::HasMethodsInventory>::Methods>
+                    .into_iter()
+                    .flat_map(pyo3::class::impl_::PyMethodsInventory::get)
+            },
+        ),
+    };
 
     let base = &attr.base;
     let flags = &attr.flags;
@@ -429,9 +445,8 @@ fn impl_class(
             fn for_each_method_def(visitor: impl FnMut(&pyo3::class::PyMethodDefType)) {
                 use pyo3::class::impl_::*;
                 let collector = PyClassImplCollector::<Self>::new();
-                pyo3::inventory::iter::<<Self as pyo3::class::methods::HasMethodsInventory>::Methods>
-                    .into_iter()
-                    .flat_map(pyo3::class::methods::PyMethodsInventory::get)
+                #iter_py_methods
+                    .chain(collector.py_class_descriptors())
                     .chain(collector.object_protocol_methods())
                     .chain(collector.async_protocol_methods())
                     .chain(collector.context_protocol_methods())
@@ -513,10 +528,12 @@ fn impl_descriptors(
         .collect::<syn::Result<_>>()?;
 
     Ok(quote! {
-        pyo3::inventory::submit! {
-            #![crate = pyo3] {
-                type Inventory = <#cls as pyo3::class::methods::HasMethodsInventory>::Methods;
-                <Inventory as pyo3::class::methods::PyMethodsInventory>::new(vec![#(#py_methods),*])
+        impl pyo3::class::impl_::PyClassDescriptors<#cls>
+            for pyo3::class::impl_::PyClassImplCollector<#cls>
+        {
+            fn py_class_descriptors(self) -> &'static [pyo3::class::methods::PyMethodDefType] {
+                static METHODS: &[pyo3::class::methods::PyMethodDefType] = &[#(#py_methods),*];
+                METHODS
             }
         }
     })