diff --git a/CHANGELOG.md b/CHANGELOG.md index f6bcbca6bc9..053b7c59ba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Add `PyList::get_item_unchecked()` and `PyTuple::get_item_unchecked()` to get items without bounds checks. [#1733](https://github.com/PyO3/pyo3/pull/1733) +- Support `#[doc = include_str!(...)]` annotations on Rust 1.54 and up. [#1746](https://github.com/PyO3/pyo3/issues/1746) - Add `PyAny::py()` as a convenience for `PyNativeType::py()`. [#1751](https://github.com/PyO3/pyo3/pull/1751) - Implement `PyString.data()` to access the raw bytes storing a Python string. [#1794](https://github.com/PyO3/pyo3/issues/1794) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 603f8df6428..a6444bba5ff 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -4,7 +4,7 @@ use crate::attributes::TextSignatureAttribute; use crate::params::{accept_args_kwargs, impl_arg_params}; use crate::pyfunction::PyFunctionOptions; use crate::pyfunction::{PyFunctionArgPyO3Attributes, PyFunctionSignature}; -use crate::utils; +use crate::utils::{self, PythonDoc}; use crate::{deprecations::Deprecations, pyfunction::Argument}; use proc_macro2::{Span, TokenStream}; use quote::ToTokens; @@ -202,7 +202,7 @@ pub struct FnSpec<'a> { pub attrs: Vec, pub args: Vec>, pub output: syn::Type, - pub doc: syn::LitStr, + pub doc: PythonDoc, pub deprecations: Deprecations, pub convention: CallingConvention, } @@ -271,7 +271,7 @@ impl<'a> FnSpec<'a> { .text_signature .as_ref() .map(|attr| (&python_name, attr)), - )?; + ); let arguments: Vec<_> = if skip_first_arg { sig.inputs @@ -601,8 +601,8 @@ fn parse_method_attributes( } for attr in attrs.drain(..) { - match attr.parse_meta()? { - syn::Meta::Path(name) => { + match attr.parse_meta() { + Ok(syn::Meta::Path(name)) => { if name.is_ident("new") || name.is_ident("__new__") { set_ty!(MethodTypeAttribute::New, name); } else if name.is_ident("init") || name.is_ident("__init__") { @@ -630,9 +630,9 @@ fn parse_method_attributes( new_attrs.push(attr) } } - syn::Meta::List(syn::MetaList { + Ok(syn::Meta::List(syn::MetaList { path, mut nested, .. - }) => { + })) => { if path.is_ident("new") { set_ty!(MethodTypeAttribute::New, path); } else if path.is_ident("init") { @@ -688,7 +688,7 @@ fn parse_method_attributes( new_attrs.push(attr) } } - syn::Meta::NameValue(_) => new_attrs.push(attr), + Ok(syn::Meta::NameValue(_)) | Err(_) => new_attrs.push(attr), } } diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index e4efe4fa8f0..e22bfbd7827 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -5,6 +5,7 @@ use crate::{ attributes::{self, take_pyo3_options}, deprecations::Deprecations, pyfunction::{impl_wrap_pyfunction, PyFunctionOptions}, + utils::PythonDoc, }; use crate::{ attributes::{is_attribute_ident, take_attributes, NameAttribute}, @@ -62,11 +63,10 @@ impl PyModuleOptions { /// Generates the function that is called by the python interpreter to initialize the native /// module -pub fn py_init(fnname: &Ident, options: PyModuleOptions, doc: syn::LitStr) -> TokenStream { +pub fn py_init(fnname: &Ident, options: PyModuleOptions, doc: PythonDoc) -> TokenStream { let name = options.name.unwrap_or_else(|| fnname.unraw()); let deprecations = options.deprecations; let cb_name = Ident::new(&format!("PyInit_{}", name), Span::call_site()); - assert!(doc.value().ends_with('\0')); quote! { #[no_mangle] diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index de80c2ccf41..034d17843f4 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -7,7 +7,7 @@ use crate::attributes::{ use crate::deprecations::Deprecations; use crate::pyimpl::PyClassMethodsType; use crate::pymethod::{impl_py_getter_def, impl_py_setter_def, PropertyType}; -use crate::utils::{self, unwrap_group}; +use crate::utils::{self, unwrap_group, PythonDoc}; use proc_macro2::{Span, TokenStream}; use quote::quote; use syn::ext::IdentExt; @@ -230,7 +230,7 @@ pub fn build_py_class( .text_signature .as_ref() .map(|attr| (get_class_python_name(&class.ident, args), attr)), - )?; + ); ensure_spanned!( class.generics.params.is_empty(), @@ -371,7 +371,7 @@ fn get_class_python_name<'a>(cls: &'a syn::Ident, attr: &'a PyClassArgs) -> &'a fn impl_class( cls: &syn::Ident, attr: &PyClassArgs, - doc: syn::LitStr, + doc: PythonDoc, field_options: Vec<(&syn::Field, FieldPyO3Options)>, methods_type: PyClassMethodsType, deprecations: Deprecations, diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index ccc37611d93..6d0c0125038 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -406,7 +406,7 @@ pub fn impl_wrap_pyfunction( .text_signature .as_ref() .map(|attr| (&python_name, attr)), - )?; + ); let function_wrapper_ident = function_wrapper_ident(&func.sig.ident); diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 0991be14998..946247e0955 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -3,7 +3,7 @@ use std::borrow::Cow; use crate::attributes::NameAttribute; -use crate::utils::ensure_not_async_fn; +use crate::utils::{ensure_not_async_fn, PythonDoc}; use crate::{deprecations::Deprecations, utils}; use crate::{ method::{FnArg, FnSpec, FnType, SelfType}, @@ -355,12 +355,10 @@ impl PropertyType<'_> { } } - fn doc(&self) -> Cow { + fn doc(&self) -> Cow { match self { PropertyType::Descriptor { field, .. } => { - let doc = utils::get_doc(&field.attrs, None) - .unwrap_or_else(|_| syn::LitStr::new("", Span::call_site())); - Cow::Owned(doc) + Cow::Owned(utils::get_doc(&field.attrs, None)) } PropertyType::Function { spec, .. } => Cow::Borrowed(&spec.doc), } diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index 93fc2afac73..33ba2c04f0f 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -1,5 +1,6 @@ // Copyright (c) 2017-present PyO3 Project and Contributors -use proc_macro2::Span; +use proc_macro2::{Span, TokenStream}; +use quote::ToTokens; use syn::spanned::Spanned; use crate::attributes::TextSignatureAttribute; @@ -54,59 +55,94 @@ pub fn option_type_argument(ty: &syn::Type) -> Option<&syn::Type> { None } -// Returns a null-terminated syn::LitStr for use as a Python docstring. +/// A syntax tree which evaluates to a null-terminated docstring for Python. +/// +/// It's built as a `concat!` evaluation, so it's hard to do anything with this +/// contents such as parse the string contents. +#[derive(Clone)] +pub struct PythonDoc(TokenStream); + +// TODO(#1782) use strip_prefix on Rust 1.45 or greater +#[allow(clippy::manual_strip)] +/// Collects all #[doc = "..."] attributes into a TokenStream evaluating to a null-terminated string +/// e.g. concat!("...", "\n", "\0") pub fn get_doc( attrs: &[syn::Attribute], text_signature: Option<(&syn::Ident, &TextSignatureAttribute)>, -) -> syn::Result { - let mut doc = String::new(); - let mut span = Span::call_site(); - - if let Some((python_name, text_signature)) = text_signature { - // create special doc string lines to set `__text_signature__` - doc.push_str(&python_name.to_string()); - span = text_signature.lit.span(); - doc.push_str(&text_signature.lit.value()); - doc.push_str("\n--\n\n"); - } - - let mut separator = ""; - let mut first = true; +) -> PythonDoc { + let mut tokens = TokenStream::new(); + let comma = syn::token::Comma(Span::call_site()); + let newline = syn::LitStr::new("\n", Span::call_site()); + + syn::Ident::new("concat", Span::call_site()).to_tokens(&mut tokens); + syn::token::Bang(Span::call_site()).to_tokens(&mut tokens); + syn::token::Bracket(Span::call_site()).surround(&mut tokens, |tokens| { + if let Some((python_name, text_signature)) = text_signature { + // create special doc string lines to set `__text_signature__` + let signature_lines = format!( + "{}{}\n--\n\n", + python_name.to_string(), + text_signature.lit.value() + ); + signature_lines.to_tokens(tokens); + comma.to_tokens(tokens); + } - for attr in attrs.iter() { - if attr.path.is_ident("doc") { - if let Ok(DocArgs { _eq_token, lit_str }) = syn::parse2(attr.tokens.clone()) { - if first { - first = false; - span = lit_str.span(); + let mut first = true; + + for attr in attrs.iter() { + if attr.path.is_ident("doc") { + if let Ok(DocArgs { + _eq_token, + token_stream, + }) = syn::parse2(attr.tokens.clone()) + { + if !first { + newline.to_tokens(tokens); + comma.to_tokens(tokens); + } else { + first = false; + } + if let Ok(syn::Lit::Str(lit_str)) = syn::parse2(token_stream.clone()) { + // Strip single left space from literal strings, if needed. + // e.g. `/// Hello world` expands to #[doc = " Hello world"] + let doc_line = lit_str.value(); + if doc_line.starts_with(' ') { + syn::LitStr::new(&doc_line[1..], lit_str.span()).to_tokens(tokens) + } else { + lit_str.to_tokens(tokens) + } + } else { + // This is probably a macro doc from Rust 1.54, e.g. #[doc = include_str!(...)] + token_stream.to_tokens(tokens) + } + comma.to_tokens(tokens); } - let d = lit_str.value(); - doc.push_str(separator); - if d.starts_with(' ') { - doc.push_str(&d[1..d.len()]); - } else { - doc.push_str(&d); - }; - separator = "\n"; } } - } - doc.push('\0'); + syn::LitStr::new("\0", Span::call_site()).to_tokens(tokens); + }); - Ok(syn::LitStr::new(&doc, span)) + PythonDoc(tokens) +} + +impl quote::ToTokens for PythonDoc { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.0.to_tokens(tokens) + } } struct DocArgs { _eq_token: syn::Token![=], - lit_str: syn::LitStr, + token_stream: TokenStream, } impl syn::parse::Parse for DocArgs { fn parse(input: syn::parse::ParseStream) -> syn::Result { let this = Self { _eq_token: input.parse()?, - lit_str: input.parse()?, + token_stream: input.parse()?, }; ensure_spanned!(input.is_empty(), input.span() => "expected end of doc attribute"); Ok(this) diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index 16742feff75..bd72cd3c9e2 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -52,10 +52,7 @@ pub fn pymodule(attr: TokenStream, input: TokenStream) -> TokenStream { return err.to_compile_error().into(); } - let doc = match get_doc(&ast.attrs, None) { - Ok(doc) => doc, - Err(err) => return err.to_compile_error().into(), - }; + let doc = get_doc(&ast.attrs, None); let expanded = py_init(&ast.sig.ident, options, doc); diff --git a/tests/not_msrv/requires_1_54.rs b/tests/not_msrv/requires_1_54.rs new file mode 100644 index 00000000000..8a68fdee428 --- /dev/null +++ b/tests/not_msrv/requires_1_54.rs @@ -0,0 +1,36 @@ +use pyo3::prelude::*; +use pyo3::types::IntoPyDict; + +#[macro_use] +#[path = "../common.rs"] +mod common; + +#[pyclass] +/// The MacroDocs class. +#[doc = concat!("Some macro ", "class ", "docs.")] +/// A very interesting type! +struct MacroDocs {} + +#[pymethods] +impl MacroDocs { + #[doc = concat!("A macro ", "example.")] + /// With mixed doc types. + fn macro_doc(&self) {} +} + +#[test] +fn meth_doc() { + Python::with_gil(|py| { + let d = [("C", py.get_type::())].into_py_dict(py); + py_assert!( + py, + *d, + "C.__doc__ == 'The MacroDocs class.\\nSome macro class docs.\\nA very interesting type!'" + ); + py_assert!( + py, + *d, + "C.macro_doc.__doc__ == 'A macro example.\\nWith mixed doc types.'" + ); + }); +} diff --git a/tests/test_not_msrv.rs b/tests/test_not_msrv.rs new file mode 100644 index 00000000000..d432c98012a --- /dev/null +++ b/tests/test_not_msrv.rs @@ -0,0 +1,10 @@ +//! Functionality which is not only not supported on MSRV, +//! but can't even be cfg-ed out on MSRV because the compiler doesn't support +//! the syntax. + +// TODO(#1782) rustversion attribute can't go on modules until Rust 1.42, so this +// funky dance has to happen... +mod requires_1_54 { + #[rustversion::since(1.54)] + include!("not_msrv/requires_1_54.rs"); +}