Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move JsonSerializable to json module (#2300) #2595

Merged
merged 1 commit into from Aug 27, 2022
Merged

Move JsonSerializable to json module (#2300) #2595

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions arrow/src/datatypes/mod.rs
View file
Open in desktop
Expand Up @@ -50,6 +50,7 @@ pub type SchemaRef = Arc<Schema>;
mod tests {
use super::*;
use crate::error::Result;
use crate::json::JsonSerializable;
use serde_json::Value::{Bool, Number as VNumber, String as VString};
use serde_json::{Number, Value};
use std::{
Expand Down
90 changes: 1 addition & 89 deletions arrow/src/datatypes/native.rs
View file
Open in desktop
Expand Up @@ -17,17 +17,11 @@

use super::DataType;
use half::f16;
use serde_json::{Number, Value};

mod private {
pub trait Sealed {}
}

/// Trait declaring any type that is serializable to JSON. This includes all primitive types (bool, i32, etc.).
pub trait JsonSerializable: 'static {
fn into_json_value(self) -> Option<Value>;
}

/// Trait expressing a Rust type that has the same in-memory representation
/// as Arrow. This includes `i16`, `f32`, but excludes `bool` (which in arrow is represented in bits).
///
Expand Down Expand Up @@ -58,8 +52,8 @@ pub trait ArrowNativeType:
+ PartialOrd
+ std::str::FromStr
+ Default
+ JsonSerializable
+ private::Sealed
+ 'static
{
/// Convert native type from usize.
#[inline]
Expand Down Expand Up @@ -120,18 +114,6 @@ pub trait ArrowPrimitiveType: 'static {
}
}

impl JsonSerializable for bool {
fn into_json_value(self) -> Option<Value> {
Some(self.into())
}
}

impl JsonSerializable for i8 {
fn into_json_value(self) -> Option<Value> {
Some(self.into())
}
}

impl private::Sealed for i8 {}
impl ArrowNativeType for i8 {
#[inline]
Expand All @@ -150,12 +132,6 @@ impl ArrowNativeType for i8 {
}
}

impl JsonSerializable for i16 {
fn into_json_value(self) -> Option<Value> {
Some(self.into())
}
}

impl private::Sealed for i16 {}
impl ArrowNativeType for i16 {
#[inline]
Expand All @@ -174,12 +150,6 @@ impl ArrowNativeType for i16 {
}
}

impl JsonSerializable for i32 {
fn into_json_value(self) -> Option<Value> {
Some(self.into())
}
}

impl private::Sealed for i32 {}
impl ArrowNativeType for i32 {
#[inline]
Expand All @@ -204,12 +174,6 @@ impl ArrowNativeType for i32 {
}
}

impl JsonSerializable for i64 {
fn into_json_value(self) -> Option<Value> {
Some(Value::Number(Number::from(self)))
}
}

impl private::Sealed for i64 {}
impl ArrowNativeType for i64 {
#[inline]
Expand All @@ -234,16 +198,6 @@ impl ArrowNativeType for i64 {
}
}

impl JsonSerializable for i128 {
fn into_json_value(self) -> Option<Value> {
// Serialize as string to avoid issues with arbitrary_precision serde_json feature
// - https://github.com/serde-rs/json/issues/559
// - https://github.com/serde-rs/json/issues/845
// - https://github.com/serde-rs/json/issues/846
Some(self.to_string().into())
}
}

impl private::Sealed for i128 {}
impl ArrowNativeType for i128 {
#[inline]
Expand All @@ -268,12 +222,6 @@ impl ArrowNativeType for i128 {
}
}

impl JsonSerializable for u8 {
fn into_json_value(self) -> Option<Value> {
Some(self.into())
}
}

impl private::Sealed for u8 {}
impl ArrowNativeType for u8 {
#[inline]
Expand All @@ -292,12 +240,6 @@ impl ArrowNativeType for u8 {
}
}

impl JsonSerializable for u16 {
fn into_json_value(self) -> Option<Value> {
Some(self.into())
}
}

impl private::Sealed for u16 {}
impl ArrowNativeType for u16 {
#[inline]
Expand All @@ -316,12 +258,6 @@ impl ArrowNativeType for u16 {
}
}

impl JsonSerializable for u32 {
fn into_json_value(self) -> Option<Value> {
Some(self.into())
}
}

impl private::Sealed for u32 {}
impl ArrowNativeType for u32 {
#[inline]
Expand All @@ -340,12 +276,6 @@ impl ArrowNativeType for u32 {
}
}

impl JsonSerializable for u64 {
fn into_json_value(self) -> Option<Value> {
Some(self.into())
}
}

impl private::Sealed for u64 {}
impl ArrowNativeType for u64 {
#[inline]
Expand All @@ -364,24 +294,6 @@ impl ArrowNativeType for u64 {
}
}

impl JsonSerializable for f16 {
fn into_json_value(self) -> Option<Value> {
Number::from_f64(f64::round(f64::from(self) * 1000.0) / 1000.0).map(Value::Number)
}
}

impl JsonSerializable for f32 {
fn into_json_value(self) -> Option<Value> {
Number::from_f64(f64::round(self as f64 * 1000.0) / 1000.0).map(Value::Number)
}
}

impl JsonSerializable for f64 {
fn into_json_value(self) -> Option<Value> {
Number::from_f64(self).map(Value::Number)
}
}

impl ArrowNativeType for f16 {}
impl private::Sealed for f16 {}
impl ArrowNativeType for f32 {}
Expand Down
55 changes: 55 additions & 0 deletions arrow/src/json/mod.rs
View file
Open in desktop
Expand Up @@ -25,3 +25,58 @@ pub mod writer;
pub use self::reader::Reader;
pub use self::reader::ReaderBuilder;
pub use self::writer::{ArrayWriter, LineDelimitedWriter, Writer};
use half::f16;
use serde_json::{Number, Value};

/// Trait declaring any type that is serializable to JSON. This includes all primitive types (bool, i32, etc.).
pub trait JsonSerializable: 'static {
fn into_json_value(self) -> Option<Value>;
}

macro_rules! json_serializable {
($t:ty) => {
impl JsonSerializable for $t {
fn into_json_value(self) -> Option<Value> {
Some(self.into())
}
}
};
}

json_serializable!(bool);
json_serializable!(u8);
json_serializable!(u16);
json_serializable!(u32);
json_serializable!(u64);
json_serializable!(i8);
json_serializable!(i16);
json_serializable!(i32);
json_serializable!(i64);

impl JsonSerializable for i128 {
fn into_json_value(self) -> Option<Value> {
// Serialize as string to avoid issues with arbitrary_precision serde_json feature
// - https://github.com/serde-rs/json/issues/559
// - https://github.com/serde-rs/json/issues/845
// - https://github.com/serde-rs/json/issues/846
Some(self.to_string().into())
}
}

impl JsonSerializable for f16 {
fn into_json_value(self) -> Option<Value> {
Number::from_f64(f64::round(f64::from(self) * 1000.0) / 1000.0).map(Value::Number)
}
}

impl JsonSerializable for f32 {
fn into_json_value(self) -> Option<Value> {
Number::from_f64(f64::round(self as f64 * 1000.0) / 1000.0).map(Value::Number)
}
}

impl JsonSerializable for f64 {
fn into_json_value(self) -> Option<Value> {
Number::from_f64(self).map(Value::Number)
}
}
16 changes: 11 additions & 5 deletions arrow/src/json/writer.rs
View file
Open in desktop
Expand Up @@ -111,11 +111,14 @@ use serde_json::Value;
use crate::array::*;
use crate::datatypes::*;
use crate::error::{ArrowError, Result};
use crate::json::JsonSerializable;
use crate::record_batch::RecordBatch;

fn primitive_array_to_json<T: ArrowPrimitiveType>(
array: &ArrayRef,
) -> Result<Vec<Value>> {
fn primitive_array_to_json<T>(array: &ArrayRef) -> Result<Vec<Value>>
where
T: ArrowPrimitiveType,
T::Native: JsonSerializable,
{
Ok(as_primitive_array::<T>(array)
.iter()
.map(|maybe_value| match maybe_value {
Expand Down Expand Up @@ -239,12 +242,15 @@ macro_rules! set_temporal_column_by_array_type {
};
}

fn set_column_by_primitive_type<T: ArrowPrimitiveType>(
fn set_column_by_primitive_type<T>(
rows: &mut [JsonMap<String, Value>],
row_count: usize,
array: &ArrayRef,
col_name: &str,
) {
) where
T: ArrowPrimitiveType,
T::Native: JsonSerializable,
{
let primitive_arr = as_primitive_array::<T>(array);

rows.iter_mut()
Expand Down