Merge pull request #1746 from davidhewitt/doc-attributes-1.54

pyo3-macros-backend: support macros inside doc attributes

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 
 - Add `PyList::get_item_unchecked` and `PyTuple::get_item_unchecked` to get items without bounds checks. [#1733](https://github.com/PyO3/pyo3/pull/1733)
+- Support `#[doc = include_str!(...)]` attributes on Rust 1.54 and up. [#1746](https://github.com/PyO3/pyo3/issues/1746)
 - Add `PyAny::py` as a convenience for `PyNativeType::py`. [#1751](https://github.com/PyO3/pyo3/pull/1751)
 - Add implementation of `std::ops::Index<usize>` for `PyList`, `PyTuple` and `PySequence`. [#1825](https://github.com/PyO3/pyo3/pull/1825)
 - Add range indexing implementations of `std::ops::Index` for `PyList`, `PyTuple` and `PySequence`. [#1829](https://github.com/PyO3/pyo3/pull/1829)

pyo3-macros-backend/src/method.rs

@@ -4,7 +4,7 @@ use crate::attributes::TextSignatureAttribute;
 use crate::params::{accept_args_kwargs, impl_arg_params};
 use crate::pyfunction::PyFunctionOptions;
 use crate::pyfunction::{PyFunctionArgPyO3Attributes, PyFunctionSignature};
-use crate::utils;
+use crate::utils::{self, PythonDoc};
 use crate::{deprecations::Deprecations, pyfunction::Argument};
 use proc_macro2::{Span, TokenStream};
 use quote::ToTokens;
@@ -202,7 +202,7 @@ pub struct FnSpec<'a> {
     pub attrs: Vec<Argument>,
     pub args: Vec<FnArg<'a>>,
     pub output: syn::Type,
-    pub doc: syn::LitStr,
+    pub doc: PythonDoc,
     pub deprecations: Deprecations,
     pub convention: CallingConvention,
 }
@@ -271,7 +271,7 @@ impl<'a> FnSpec<'a> {
                 .text_signature
                 .as_ref()
                 .map(|attr| (&python_name, attr)),
-        )?;
+        );
 
         let arguments: Vec<_> = if skip_first_arg {
             sig.inputs
@@ -602,8 +602,8 @@ fn parse_method_attributes(
     }
 
     for attr in attrs.drain(..) {
-        match attr.parse_meta()? {
-            syn::Meta::Path(name) => {
+        match attr.parse_meta() {
+            Ok(syn::Meta::Path(name)) => {
                 if name.is_ident("new") || name.is_ident("__new__") {
                     set_ty!(MethodTypeAttribute::New, name);
                 } else if name.is_ident("init") || name.is_ident("__init__") {
@@ -631,9 +631,9 @@ fn parse_method_attributes(
                     new_attrs.push(attr)
                 }
             }
-            syn::Meta::List(syn::MetaList {
+            Ok(syn::Meta::List(syn::MetaList {
                 path, mut nested, ..
-            }) => {
+            })) => {
                 if path.is_ident("new") {
                     set_ty!(MethodTypeAttribute::New, path);
                 } else if path.is_ident("init") {
@@ -689,7 +689,7 @@ fn parse_method_attributes(
                     new_attrs.push(attr)
                 }
             }
-            syn::Meta::NameValue(_) => new_attrs.push(attr),
+            Ok(syn::Meta::NameValue(_)) | Err(_) => new_attrs.push(attr),
         }
     }
 

pyo3-macros-backend/src/module.rs

@@ -5,6 +5,7 @@ use crate::{
     attributes::{self, take_pyo3_options},
     deprecations::Deprecations,
     pyfunction::{impl_wrap_pyfunction, PyFunctionOptions},
+    utils::PythonDoc,
 };
 use crate::{
     attributes::{is_attribute_ident, take_attributes, NameAttribute},
@@ -62,11 +63,10 @@ impl PyModuleOptions {
 
 /// Generates the function that is called by the python interpreter to initialize the native
 /// module
-pub fn py_init(fnname: &Ident, options: PyModuleOptions, doc: syn::LitStr) -> TokenStream {
+pub fn py_init(fnname: &Ident, options: PyModuleOptions, doc: PythonDoc) -> TokenStream {
     let name = options.name.unwrap_or_else(|| fnname.unraw());
     let deprecations = options.deprecations;
     let cb_name = Ident::new(&format!("PyInit_{}", name), Span::call_site());
-    assert!(doc.value().ends_with('\0'));
 
     quote! {
         #[no_mangle]

pyo3-macros-backend/src/pyclass.rs

@@ -7,7 +7,7 @@ use crate::attributes::{
 use crate::deprecations::Deprecations;
 use crate::pyimpl::PyClassMethodsType;
 use crate::pymethod::{impl_py_getter_def, impl_py_setter_def, PropertyType};
-use crate::utils::{self, unwrap_group};
+use crate::utils::{self, unwrap_group, PythonDoc};
 use proc_macro2::{Span, TokenStream};
 use quote::quote;
 use syn::ext::IdentExt;
@@ -230,7 +230,7 @@ pub fn build_py_class(
             .text_signature
             .as_ref()
             .map(|attr| (get_class_python_name(&class.ident, args), attr)),
-    )?;
+    );
 
     ensure_spanned!(
         class.generics.params.is_empty(),
@@ -371,7 +371,7 @@ fn get_class_python_name<'a>(cls: &'a syn::Ident, attr: &'a PyClassArgs) -> &'a
 fn impl_class(
     cls: &syn::Ident,
     attr: &PyClassArgs,
-    doc: syn::LitStr,
+    doc: PythonDoc,
     field_options: Vec<(&syn::Field, FieldPyO3Options)>,
     methods_type: PyClassMethodsType,
     deprecations: Deprecations,

pyo3-macros-backend/src/pyfunction.rs

@@ -406,7 +406,7 @@ pub fn impl_wrap_pyfunction(
             .text_signature
             .as_ref()
             .map(|attr| (&python_name, attr)),
-    )?;
+    );
 
     let function_wrapper_ident = function_wrapper_ident(&func.sig.ident);
 

pyo3-macros-backend/src/pymethod.rs

@@ -3,7 +3,7 @@
 use std::borrow::Cow;
 
 use crate::attributes::NameAttribute;
-use crate::utils::ensure_not_async_fn;
+use crate::utils::{ensure_not_async_fn, PythonDoc};
 use crate::{deprecations::Deprecations, utils};
 use crate::{
     method::{FnArg, FnSpec, FnType, SelfType},
@@ -355,12 +355,10 @@ impl PropertyType<'_> {
         }
     }
 
-    fn doc(&self) -> Cow<syn::LitStr> {
+    fn doc(&self) -> Cow<PythonDoc> {
         match self {
             PropertyType::Descriptor { field, .. } => {
-                let doc = utils::get_doc(&field.attrs, None)
-                    .unwrap_or_else(|_| syn::LitStr::new("", Span::call_site()));
-                Cow::Owned(doc)
+                Cow::Owned(utils::get_doc(&field.attrs, None))
             }
             PropertyType::Function { spec, .. } => Cow::Borrowed(&spec.doc),
         }

pyo3-macros-backend/src/utils.rs

@@ -1,5 +1,6 @@
 // Copyright (c) 2017-present PyO3 Project and Contributors
-use proc_macro2::Span;
+use proc_macro2::{Span, TokenStream};
+use quote::ToTokens;
 use syn::spanned::Spanned;
 
 use crate::attributes::TextSignatureAttribute;
@@ -54,59 +55,94 @@ pub fn option_type_argument(ty: &syn::Type) -> Option<&syn::Type> {
     None
 }
 
-// Returns a null-terminated syn::LitStr for use as a Python docstring.
+/// A syntax tree which evaluates to a null-terminated docstring for Python.
+///
+/// It's built as a `concat!` evaluation, so it's hard to do anything with this
+/// contents such as parse the string contents.
+#[derive(Clone)]
+pub struct PythonDoc(TokenStream);
+
+// TODO(#1782) use strip_prefix on Rust 1.45 or greater
+#[allow(clippy::manual_strip)]
+/// Collects all #[doc = "..."] attributes into a TokenStream evaluating to a null-terminated string
+/// e.g. concat!("...", "\n", "\0")
 pub fn get_doc(
     attrs: &[syn::Attribute],
     text_signature: Option<(&syn::Ident, &TextSignatureAttribute)>,
-) -> syn::Result<syn::LitStr> {
-    let mut doc = String::new();
-    let mut span = Span::call_site();
-
-    if let Some((python_name, text_signature)) = text_signature {
-        // create special doc string lines to set `__text_signature__`
-        doc.push_str(&python_name.to_string());
-        span = text_signature.lit.span();
-        doc.push_str(&text_signature.lit.value());
-        doc.push_str("\n--\n\n");
-    }
-
-    let mut separator = "";
-    let mut first = true;
+) -> PythonDoc {
+    let mut tokens = TokenStream::new();
+    let comma = syn::token::Comma(Span::call_site());
+    let newline = syn::LitStr::new("\n", Span::call_site());
+
+    syn::Ident::new("concat", Span::call_site()).to_tokens(&mut tokens);
+    syn::token::Bang(Span::call_site()).to_tokens(&mut tokens);
+    syn::token::Bracket(Span::call_site()).surround(&mut tokens, |tokens| {
+        if let Some((python_name, text_signature)) = text_signature {
+            // create special doc string lines to set `__text_signature__`
+            let signature_lines = format!(
+                "{}{}\n--\n\n",
+                python_name.to_string(),
+                text_signature.lit.value()
+            );
+            signature_lines.to_tokens(tokens);
+            comma.to_tokens(tokens);
+        }
 
-    for attr in attrs.iter() {
-        if attr.path.is_ident("doc") {
-            if let Ok(DocArgs { _eq_token, lit_str }) = syn::parse2(attr.tokens.clone()) {
-                if first {
-                    first = false;
-                    span = lit_str.span();
+        let mut first = true;
+
+        for attr in attrs.iter() {
+            if attr.path.is_ident("doc") {
+                if let Ok(DocArgs {
+                    _eq_token,
+                    token_stream,
+                }) = syn::parse2(attr.tokens.clone())
+                {
+                    if !first {
+                        newline.to_tokens(tokens);
+                        comma.to_tokens(tokens);
+                    } else {
+                        first = false;
+                    }
+                    if let Ok(syn::Lit::Str(lit_str)) = syn::parse2(token_stream.clone()) {
+                        // Strip single left space from literal strings, if needed.
+                        // e.g. `/// Hello world` expands to #[doc = " Hello world"]
+                        let doc_line = lit_str.value();
+                        if doc_line.starts_with(' ') {
+                            syn::LitStr::new(&doc_line[1..], lit_str.span()).to_tokens(tokens)
+                        } else {
+                            lit_str.to_tokens(tokens)
+                        }
+                    } else {
+                        // This is probably a macro doc from Rust 1.54, e.g. #[doc = include_str!(...)]
+                        token_stream.to_tokens(tokens)
+                    }
+                    comma.to_tokens(tokens);
                 }
-                let d = lit_str.value();
-                doc.push_str(separator);
-                if d.starts_with(' ') {
-                    doc.push_str(&d[1..d.len()]);
-                } else {
-                    doc.push_str(&d);
-                };
-                separator = "\n";
             }
         }
-    }
 
-    doc.push('\0');
+        syn::LitStr::new("\0", Span::call_site()).to_tokens(tokens);
+    });
 
-    Ok(syn::LitStr::new(&doc, span))
+    PythonDoc(tokens)
+}
+
+impl quote::ToTokens for PythonDoc {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        self.0.to_tokens(tokens)
+    }
 }
 
 struct DocArgs {
     _eq_token: syn::Token![=],
-    lit_str: syn::LitStr,
+    token_stream: TokenStream,
 }
 
 impl syn::parse::Parse for DocArgs {
     fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
         let this = Self {
             _eq_token: input.parse()?,
-            lit_str: input.parse()?,
+            token_stream: input.parse()?,
         };
         ensure_spanned!(input.is_empty(), input.span() => "expected end of doc attribute");
         Ok(this)

pyo3-macros/src/lib.rs

@@ -52,10 +52,7 @@ pub fn pymodule(attr: TokenStream, input: TokenStream) -> TokenStream {
         return err.to_compile_error().into();
     }
 
-    let doc = match get_doc(&ast.attrs, None) {
-        Ok(doc) => doc,
-        Err(err) => return err.to_compile_error().into(),
-    };
+    let doc = get_doc(&ast.attrs, None);
 
     let expanded = py_init(&ast.sig.ident, options, doc);
 

tests/not_msrv/requires_1_54.rs

@@ -0,0 +1,36 @@
+use pyo3::prelude::*;
+use pyo3::types::IntoPyDict;
+
+#[macro_use]
+#[path = "../common.rs"]
+mod common;
+
+#[pyclass]
+/// The MacroDocs class.
+#[doc = concat!("Some macro ", "class ", "docs.")]
+/// A very interesting type!
+struct MacroDocs {}
+
+#[pymethods]
+impl MacroDocs {
+    #[doc = concat!("A macro ", "example.")]
+    /// With mixed doc types.
+    fn macro_doc(&self) {}
+}
+
+#[test]
+fn meth_doc() {
+    Python::with_gil(|py| {
+        let d = [("C", py.get_type::<MacroDocs>())].into_py_dict(py);
+        py_assert!(
+            py,
+            *d,
+            "C.__doc__ == 'The MacroDocs class.\\nSome macro class docs.\\nA very interesting type!'"
+        );
+        py_assert!(
+            py,
+            *d,
+            "C.macro_doc.__doc__ == 'A macro example.\\nWith mixed doc types.'"
+        );
+    });
+}

tests/test_not_msrv.rs

@@ -0,0 +1,10 @@
+//! Functionality which is not only not supported on MSRV,
+//! but can't even be cfg-ed out on MSRV because the compiler doesn't support
+//! the syntax.
+
+// TODO(#1782) rustversion attribute can't go on modules until Rust 1.42, so this
+// funky dance has to happen...
+mod requires_1_54 {
+    #[rustversion::since(1.54)]
+    include!("not_msrv/requires_1_54.rs");
+}