Merge pull request #1619 from birkenfeld/fastcall

Implement METH_FASTCALL for pyfunctions.

CHANGELOG.md

@@ -48,6 +48,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Reduce LLVM line counts to improve compilation times. [#1604](https://github.com/PyO3/pyo3/pull/1604)
 - Deprecate string-literal second argument to `#[pyfn(m, "name")]`. [#1610](https://github.com/PyO3/pyo3/pull/1610)
 - No longer call `PyEval_InitThreads()` in `#[pymodule]` init code. [#1630](https://github.com/PyO3/pyo3/pull/1630)
+- Use `METH_FASTCALL` argument passing convention, when possible, to improve `#[pyfunction]` performance. [#1619](https://github.com/PyO3/pyo3/pull/1619)
 
 ### Removed
 - Remove deprecated exception names `BaseException` etc. [#1426](https://github.com/PyO3/pyo3/pull/1426)
@@ -63,6 +64,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
   - `PyModuleDef_INIT` [#1630](https://github.com/PyO3/pyo3/pull/1630)
 - Remove `__doc__` from module's `__all__`. [#1509](https://github.com/PyO3/pyo3/pull/1509)
 - Remove `PYO3_CROSS_INCLUDE_DIR` environment variable and the associated C header parsing functionality. [#1521](https://github.com/PyO3/pyo3/pull/1521)
+- Remove `raw_pycfunction!` macro. [#1619](https://github.com/PyO3/pyo3/pull/1619)
 
 ### Fixed
 - Remove FFI definition `PyCFunction_ClearFreeList` for Python 3.9 and later. [#1425](https://github.com/PyO3/pyo3/pull/1425)

guide/src/function.md

@@ -279,12 +279,11 @@ in the function body.
 
 ## Accessing the FFI functions
 
-In order to make Rust functions callable from Python, PyO3 generates a
-`extern "C" Fn(slf: *mut PyObject, args: *mut PyObject, kwargs: *mut PyObject) -> *mut Pyobject`
-function and embeds the call to the Rust function inside this FFI-wrapper function. This
-wrapper handles extraction of the regular arguments and the keyword arguments from the input
-`PyObjects`. Since this function is not user-defined but required to build a `PyCFunction`, PyO3
-offers the `raw_pycfunction!()` macro to get the identifier of this generated wrapper.
+In order to make Rust functions callable from Python, PyO3 generates an `extern "C"`
+function whose exact signature depends on the Rust signature.  (PyO3 chooses the optimal
+Python argument passing convention.) It then embeds the call to the Rust function inside this
+FFI-wrapper function. This wrapper handles extraction of the regular arguments and the keyword
+arguments from the input `PyObject`s.
 
 The `wrap_pyfunction` macro can be used to directly get a `PyCFunction` given a
 `#[pyfunction]` and a `PyModule`: `wrap_pyfunction!(rust_fun, module)`.

pyo3-macros-backend/src/method.rs

@@ -158,6 +158,28 @@ pub fn parse_method_receiver(arg: &syn::FnArg) -> syn::Result<SelfType> {
 }
 
 impl<'a> FnSpec<'a> {
+    /// Determine if the function gets passed a *args tuple or **kwargs dict.
+    pub fn accept_args_kwargs(&self) -> (bool, bool) {
+        let (mut accept_args, mut accept_kwargs) = (false, false);
+
+        for s in &self.attrs {
+            match s {
+                Argument::VarArgs(_) => accept_args = true,
+                Argument::KeywordArgs(_) => accept_kwargs = true,
+                _ => continue,
+            }
+        }
+
+        (accept_args, accept_kwargs)
+    }
+
+    /// Return true if the function can use METH_FASTCALL.
+    ///
+    /// This is true on Py3.7+, except with the stable ABI (abi3).
+    pub fn can_use_fastcall(&self) -> bool {
+        cfg!(all(Py_3_7, not(Py_LIMITED_API)))
+    }
+
     /// Parser function signature and function attributes
     pub fn parse(
         sig: &'a mut syn::Signature,

pyo3-macros-backend/src/pyfunction.rs

@@ -401,25 +401,29 @@ pub fn impl_wrap_pyfunction(
     let name = &func.sig.ident;
     let wrapper_ident = format_ident!("__pyo3_raw_{}", name);
     let wrapper = function_c_wrapper(name, &wrapper_ident, &spec, options.pass_module)?;
-    let methoddef = if spec.args.is_empty() {
-        quote!(noargs)
-    } else {
-        quote!(cfunction_with_keywords)
-    };
-    let cfunc = if spec.args.is_empty() {
-        quote!(PyCFunction)
+    let (methoddef_meth, cfunc_variant) = if spec.args.is_empty() {
+        (quote!(noargs), quote!(PyCFunction))
+    } else if spec.can_use_fastcall() {
+        (
+            quote!(fastcall_cfunction_with_keywords),
+            quote!(PyCFunctionFastWithKeywords),
+        )
     } else {
-        quote!(PyCFunctionWithKeywords)
+        (
+            quote!(cfunction_with_keywords),
+            quote!(PyCFunctionWithKeywords),
+        )
     };
+
     let wrapped_pyfunction = quote! {
         #wrapper
         pub(crate) fn #function_wrapper_ident<'a>(
             args: impl Into<pyo3::derive_utils::PyFunctionArguments<'a>>
         ) -> pyo3::PyResult<&'a pyo3::types::PyCFunction> {
             pyo3::types::PyCFunction::internal_new(
-                pyo3::class::methods::PyMethodDef:: #methoddef (
+                pyo3::class::methods::PyMethodDef:: #methoddef_meth (
                     #python_name,
-                    pyo3::class::methods:: #cfunc (#wrapper_ident),
+                    pyo3::class::methods:: #cfunc_variant (#wrapper_ident),
                     #doc,
                 ),
                 args.into(),
@@ -469,8 +473,36 @@ fn function_c_wrapper(
                 })
             }
         })
+    } else if spec.can_use_fastcall() {
+        let body = impl_arg_params(spec, None, cb, &py, true)?;
+        Ok(quote! {
+            unsafe extern "C" fn #wrapper_ident(
+                _slf: *mut pyo3::ffi::PyObject,
+                _args: *const *mut pyo3::ffi::PyObject,
+                _nargs: pyo3::ffi::Py_ssize_t,
+                _kwnames: *mut pyo3::ffi::PyObject) -> *mut pyo3::ffi::PyObject
+            {
+                pyo3::callback::handle_panic(|#py| {
+                    #slf_module
+                    // _nargs is the number of positional arguments in the _args array,
+                    // the number of KW args is given by the length of _kwnames
+                    let _kwnames: Option<&pyo3::types::PyTuple> = #py.from_borrowed_ptr_or_opt(_kwnames);
+                    // Safety: &PyAny has the same memory layout as `*mut ffi::PyObject`
+                    let _args = _args as *const &pyo3::PyAny;
+                    let _kwargs = if let Some(kwnames) = _kwnames {
+                        std::slice::from_raw_parts(_args.offset(_nargs), kwnames.len())
+                    } else {
+                        &[]
+                    };
+                    let _args = std::slice::from_raw_parts(_args, _nargs as usize);
+
+                    #body
+                })
+            }
+
+        })
     } else {
-        let body = impl_arg_params(spec, None, cb, &py)?;
+        let body = impl_arg_params(spec, None, cb, &py, false)?;
         Ok(quote! {
             unsafe extern "C" fn #wrapper_ident(
                 _slf: *mut pyo3::ffi::PyObject,
@@ -482,7 +514,6 @@ fn function_c_wrapper(
                     #slf_module
                     let _args = #py.from_borrowed_ptr::<pyo3::types::PyTuple>(_args);
                     let _kwargs: Option<&pyo3::types::PyDict> = #py.from_borrowed_ptr_or_opt(_kwargs);
-
                     #body
                 })
             }

pyo3-macros-backend/src/pymethod.rs

@@ -93,7 +93,7 @@ pub fn impl_wrap_cfunction_with_keywords(
     let body = impl_call(cls, &spec);
     let slf = self_ty.receiver(cls);
     let py = syn::Ident::new("_py", Span::call_site());
-    let body = impl_arg_params(&spec, Some(cls), body, &py)?;
+    let body = impl_arg_params(&spec, Some(cls), body, &py, false)?;
     let deprecations = &spec.deprecations;
     Ok(quote! {{
         unsafe extern "C" fn __wrap(
@@ -114,6 +114,42 @@ pub fn impl_wrap_cfunction_with_keywords(
     }})
 }
 
+/// Generate function wrapper for PyCFunctionFastWithKeywords
+pub fn impl_wrap_fastcall_cfunction_with_keywords(
+    cls: &syn::Type,
+    spec: &FnSpec<'_>,
+    self_ty: &SelfType,
+) -> Result<TokenStream> {
+    let body = impl_call(cls, &spec);
+    let slf = self_ty.receiver(cls);
+    let py = syn::Ident::new("_py", Span::call_site());
+    let body = impl_arg_params(&spec, Some(cls), body, &py, true)?;
+    Ok(quote! {{
+        unsafe extern "C" fn __wrap(
+            _slf: *mut pyo3::ffi::PyObject,
+            _args: *const *mut pyo3::ffi::PyObject,
+            _nargs: pyo3::ffi::Py_ssize_t,
+            _kwnames: *mut pyo3::ffi::PyObject) -> *mut pyo3::ffi::PyObject
+        {
+            pyo3::callback::handle_panic(|#py| {
+                #slf
+                let _kwnames: Option<&pyo3::types::PyTuple> = #py.from_borrowed_ptr_or_opt(_kwnames);
+                // Safety: &PyAny has the same memory layout as `*mut ffi::PyObject`
+                let _args = _args as *const &pyo3::PyAny;
+                let _kwargs = if let Some(kwnames) = _kwnames {
+                    std::slice::from_raw_parts(_args.offset(_nargs), kwnames.len())
+                } else {
+                    &[]
+                };
+                let _args = std::slice::from_raw_parts(_args, _nargs as usize);
+
+                #body
+            })
+        }
+        __wrap
+    }})
+}
+
 /// Generate function wrapper PyCFunction
 pub fn impl_wrap_noargs(cls: &syn::Type, spec: &FnSpec<'_>, self_ty: &SelfType) -> TokenStream {
     let body = impl_call(cls, &spec);
@@ -142,7 +178,7 @@ pub fn impl_wrap_new(cls: &syn::Type, spec: &FnSpec<'_>) -> Result<TokenStream>
     let names: Vec<syn::Ident> = get_arg_names(&spec);
     let cb = quote! { #cls::#name(#(#names),*) };
     let py = syn::Ident::new("_py", Span::call_site());
-    let body = impl_arg_params(spec, Some(cls), cb, &py)?;
+    let body = impl_arg_params(spec, Some(cls), cb, &py, false)?;
     let deprecations = &spec.deprecations;
     Ok(quote! {{
         #[allow(unused_mut)]
@@ -172,7 +208,7 @@ pub fn impl_wrap_class(cls: &syn::Type, spec: &FnSpec<'_>) -> Result<TokenStream
     let names: Vec<syn::Ident> = get_arg_names(&spec);
     let cb = quote! { pyo3::callback::convert(_py, #cls::#name(&_cls, #(#names),*)) };
     let py = syn::Ident::new("_py", Span::call_site());
-    let body = impl_arg_params(spec, Some(cls), cb, &py)?;
+    let body = impl_arg_params(spec, Some(cls), cb, &py, false)?;
     let deprecations = &spec.deprecations;
     Ok(quote! {{
         #[allow(unused_mut)]
@@ -200,7 +236,7 @@ pub fn impl_wrap_static(cls: &syn::Type, spec: &FnSpec<'_>) -> Result<TokenStrea
     let names: Vec<syn::Ident> = get_arg_names(&spec);
     let cb = quote! { pyo3::callback::convert(_py, #cls::#name(#(#names),*)) };
     let py = syn::Ident::new("_py", Span::call_site());
-    let body = impl_arg_params(spec, Some(cls), cb, &py)?;
+    let body = impl_arg_params(spec, Some(cls), cb, &py, false)?;
     let deprecations = &spec.deprecations;
     Ok(quote! {{
         #[allow(unused_mut)]
@@ -379,6 +415,7 @@ pub fn impl_arg_params(
     self_: Option<&syn::Type>,
     body: TokenStream,
     py: &syn::Ident,
+    fastcall: bool,
 ) -> Result<TokenStream> {
     if spec.args.is_empty() {
         return Ok(body);
@@ -428,16 +465,7 @@ pub fn impl_arg_params(
         )?);
     }
 
-    let (mut accept_args, mut accept_kwargs) = (false, false);
-
-    for s in spec.attrs.iter() {
-        use crate::pyfunction::Argument;
-        match s {
-            Argument::VarArgs(_) => accept_args = true,
-            Argument::KeywordArgs(_) => accept_kwargs = true,
-            _ => continue,
-        }
-    }
+    let (accept_args, accept_kwargs) = spec.accept_args_kwargs();
 
     let cls_name = if let Some(cls) = self_ {
         quote! { Some(<#cls as pyo3::type_object::PyTypeInfo>::NAME) }
@@ -446,6 +474,24 @@ pub fn impl_arg_params(
     };
     let python_name = &spec.python_name;
 
+    let (args_to_extract, kwargs_to_extract) = if fastcall {
+        // _args is a &[&PyAny], _kwnames is a Option<&PyTuple> containing the
+        // keyword names of the keyword args in _kwargs
+        (
+            // need copied() for &&PyAny -> &PyAny
+            quote! { _args.iter().copied() },
+            quote! { _kwnames.map(|kwnames| {
+                kwnames.as_slice().iter().copied().zip(_kwargs.iter().copied())
+            }) },
+        )
+    } else {
+        // _args is a &PyTuple, _kwargs is an Option<&PyDict>
+        (
+            quote! { _args.iter() },
+            quote! { _kwargs.map(|dict| dict.iter()) },
+        )
+    };
+
     // create array of arguments, and then parse
     Ok(quote! {
         {
@@ -462,7 +508,12 @@ pub fn impl_arg_params(
             };
 
             let mut #args_array = [None; #num_params];
-            let (_args, _kwargs) = DESCRIPTION.extract_arguments(_args, _kwargs, &mut #args_array)?;
+            let (_args, _kwargs) = DESCRIPTION.extract_arguments(
+                #py,
+                #args_to_extract,
+                #kwargs_to_extract,
+                &mut #args_array
+            )?;
 
             #(#param_conversion)*
 
@@ -616,32 +667,36 @@ pub fn impl_py_method_def(
     let add_flags = flags.map(|flags| quote!(.flags(#flags)));
     let python_name = spec.null_terminated_python_name();
     let doc = &spec.doc;
-    if spec.args.is_empty() {
-        let wrapper = impl_wrap_noargs(cls, spec, self_ty);
-        Ok(quote! {
-            pyo3::class::PyMethodDefType::Method({
-                pyo3::class::PyMethodDef::noargs(
-                    #python_name,
-                    pyo3::class::methods::PyCFunction(#wrapper),
-                    #doc
-                )
-                #add_flags
-
-            })
-        })
+    let (methoddef_meth, cfunc_variant) = if spec.args.is_empty() {
+        (quote!(noargs), quote!(PyCFunction))
+    } else if spec.can_use_fastcall() {
+        (
+            quote!(fastcall_cfunction_with_keywords),
+            quote!(PyCFunctionFastWithKeywords),
+        )
     } else {
-        let wrapper = impl_wrap_cfunction_with_keywords(cls, &spec, self_ty)?;
-        Ok(quote! {
-            pyo3::class::PyMethodDefType::Method({
-                pyo3::class::PyMethodDef::cfunction_with_keywords(
-                    #python_name,
-                    pyo3::class::methods::PyCFunctionWithKeywords(#wrapper),
-                    #doc
-                )
-                #add_flags
-            })
+        (
+            quote!(cfunction_with_keywords),
+            quote!(PyCFunctionWithKeywords),
+        )
+    };
+    let wrapper = if spec.args.is_empty() {
+        impl_wrap_noargs(cls, spec, self_ty)
+    } else if spec.can_use_fastcall() {
+        impl_wrap_fastcall_cfunction_with_keywords(cls, &spec, self_ty)?
+    } else {
+        impl_wrap_cfunction_with_keywords(cls, &spec, self_ty)?
+    };
+    Ok(quote! {
+        pyo3::class::PyMethodDefType::Method({
+            pyo3::class::PyMethodDef:: #methoddef_meth (
+                #python_name,
+                pyo3::class::methods:: #cfunc_variant (#wrapper),
+                #doc
+            )
+            #add_flags
         })
-    }
+    })
 }
 
 pub fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec) -> Result<TokenStream> {