deserialize_with expects newtype enum variants, even for unit enum variants

[Jonathan-Landeed]

#[derive(serde::Deserialize)]
pub enum Status {
    #[serde(deserialize_with = "my_deserializer")]
    Down,
    Success,
}

when expanded has the visitor:

#[doc(hidden)]
struct __Visitor<'de> {
    marker: _serde::__private::PhantomData<Status>,
    lifetime: _serde::__private::PhantomData<&'de ()>,
}
impl<'de> _serde::de::Visitor<'de> for __Visitor<'de> {
    type Value = Status;
    fn expecting(
        &self,
        __formatter: &mut _serde::__private::Formatter,
    ) -> _serde::__private::fmt::Result {
        _serde::__private::Formatter::write_str(
            __formatter,
            "enum Status",
        )
    }
    fn visit_enum<__A>(
        self,
        __data: __A,
    ) -> _serde::__private::Result<Self::Value, __A::Error>
    where
        __A: _serde::de::EnumAccess<'de>,
    {
        match _serde::de::EnumAccess::variant(__data)? {
            (__Field::__field0, __variant) => {
                #[doc(hidden)]
                struct __DeserializeWith<'de> {
                    value: (),
                    phantom: _serde::__private::PhantomData<Status>,
                    lifetime: _serde::__private::PhantomData<&'de ()>,
                }
                impl<'de> _serde::Deserialize<'de>
                for __DeserializeWith<'de> {
                    fn deserialize<__D>(
                        __deserializer: __D,
                    ) -> _serde::__private::Result<Self, __D::Error>
                    where
                        __D: _serde::Deserializer<'de>,
                    {
                        _serde::__private::Ok(__DeserializeWith {
                            value: my_deserializer(__deserializer)?,
                            phantom: _serde::__private::PhantomData,
                            lifetime: _serde::__private::PhantomData,
                        })
                    }
                }
                _serde::__private::Result::map(
                    _serde::de::VariantAccess::newtype_variant::<
                        __DeserializeWith<'de>,
                    >(__variant),
                    |__wrap| Status::Down,
                )
            }
            (__Field::__field1, __variant) => {
                _serde::de::VariantAccess::unit_variant(__variant)?;
                _serde::__private::Ok(Status::Success)
            }
        }
    }
}

No matter what my_deserializer is, this results in "invalid type: unit variant, expected newtype variant".
I tried

pub fn my_deserializer<'de, D, T>(deserializer: D) -> Result<T, D::Error>
where
    D: Deserializer<'de>,
    T: Deserialize<'de>,
{
    T::deserialize(deserializer)
}

and

pub fn my_deserializer'de, D>(deserializer: D) -> Result<(), D::Error>
where
    D: Deserializer<'de>,
{
    Ok(())
}

This generally wouldn't be a problem because you can omit the #[serde(deserialize_with = "my_deserializer")] on any unit variants, but I was trying to get this to work with a field_attribute in prost-build, so it was automatically being applied to every field in the enum.