Update thrift v0.16 and vendor parquet-format (#2502)

.gitattributes

@@ -1,6 +1 @@
-r/R/RcppExports.R linguist-generated=true
-r/R/arrowExports.R linguist-generated=true
-r/src/RcppExports.cpp linguist-generated=true
-r/src/arrowExports.cpp linguist-generated=true
-r/man/*.Rd linguist-generated=true
-
+parquet/src/format.rs linguist-generated

parquet/CONTRIBUTING.md

@@ -60,7 +60,18 @@ Run `cargo bench` for benchmarks.
 To build documentation, run `cargo doc --no-deps`.
 To compile and view in the browser, run `cargo doc --no-deps --open`.
 
-## Update Supported Parquet Version
+## Update Parquet Format
 
-To update Parquet format to a newer version, check if [parquet-format](https://github.com/sunchao/parquet-format-rs)
-version is available. Then simply update version of `parquet-format` crate in Cargo.toml.
+To generate the parquet format code run
+
+```
+$ git clone https://github.com/apache/thrift
+$ cd thrift
+$ git checkout v0.16.0
+# docker build just builds a docker image with thrift dependencies
+$ docker build -t thrift build/docker/ubuntu-bionic
+# build/docker/scripts/cmake.sh actually compiles thrift
+$ docker run -v $(pwd):/thrift/src -it thrift build/docker/scripts/cmake.sh && wget https://raw.githubusercontent.com/apache/parquet-format/apache-parquet-format-2.9.0/src/main/thrift/parquet.thrift && ./cmake_build/compiler/cpp/bin/thrift --gen rs parquet.thrift
+```
+
+Then copy the generated `parquet.rs` into `src/format.rs` and commit changes.

parquet/Cargo.toml

@@ -31,9 +31,8 @@ rust-version = "1.62"
 
 [dependencies]
 ahash = "0.8"
-parquet-format = { version = "4.0.0", default-features = false }
 bytes = { version = "1.1", default-features = false, features = ["std"] }
-thrift = { version = "0.13", default-features = false }
+thrift = { version = "0.16", default-features = false }
 snap = { version = "1.0", default-features = false, optional = true }
 brotli = { version = "3.3", default-features = false, features = ["std"], optional = true }
 flate2 = { version = "1.0", default-features = false, features = ["rust_backend"], optional = true }

parquet/src/arrow/arrow_reader/mod.rs

@@ -1668,7 +1668,7 @@ mod tests {
         schema: TypePtr,
         field: Option<Field>,
         opts: &TestOptions,
-    ) -> Result<parquet_format::FileMetaData> {
+    ) -> Result<crate::format::FileMetaData> {
         let mut writer_props = opts.writer_props();
         if let Some(field) = field {
             let arrow_schema = Schema::new(vec![field]);

parquet/src/arrow/arrow_reader/selection.rs

@@ -121,7 +121,7 @@ impl RowSelection {
     #[cfg(any(test, feature = "async"))]
     pub(crate) fn scan_ranges(
         &self,
-        page_locations: &[parquet_format::PageLocation],
+        page_locations: &[crate::format::PageLocation],
     ) -> Vec<Range<usize>> {
         let mut ranges = vec![];
         let mut row_offset = 0;
@@ -302,7 +302,7 @@ impl From<RowSelection> for VecDeque<RowSelector> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use parquet_format::PageLocation;
+    use crate::format::PageLocation;
     use rand::{thread_rng, Rng};
 
     #[test]

parquet/src/arrow/arrow_writer/mod.rs

@@ -230,7 +230,7 @@ impl<W: Write> ArrowWriter<W> {
     }
 
     /// Close and finalize the underlying Parquet writer
-    pub fn close(mut self) -> Result<parquet_format::FileMetaData> {
+    pub fn close(mut self) -> Result<crate::format::FileMetaData> {
         self.flush()?;
         self.writer.close()
     }

parquet/src/arrow/async_reader.rs

@@ -84,11 +84,11 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::task::{Context, Poll};
 
+use crate::format::OffsetIndex;
 use bytes::{Buf, Bytes};
 use futures::future::{BoxFuture, FutureExt};
 use futures::ready;
 use futures::stream::Stream;
-use parquet_format::OffsetIndex;
 use thrift::protocol::TCompactInputProtocol;
 
 use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt};

parquet/src/column/page.rs

@@ -20,9 +20,9 @@
 use crate::basic::{Encoding, PageType};
 use crate::errors::{ParquetError, Result};
 use crate::file::{metadata::ColumnChunkMetaData, statistics::Statistics};
+use crate::format::PageHeader;
 use crate::schema::types::{ColumnDescPtr, SchemaDescPtr};
 use crate::util::memory::ByteBufferPtr;
-use parquet_format::PageHeader;
 
 /// Parquet Page definition.
 ///
@@ -209,15 +209,15 @@ impl TryFrom<&PageHeader> for PageMetadata {
 
     fn try_from(value: &PageHeader) -> std::result::Result<Self, Self::Error> {
         match value.type_ {
-            parquet_format::PageType::DataPage => Ok(PageMetadata {
+            crate::format::PageType::DATA_PAGE => Ok(PageMetadata {
                 num_rows: value.data_page_header.as_ref().unwrap().num_values as usize,
                 is_dict: false,
             }),
-            parquet_format::PageType::DictionaryPage => Ok(PageMetadata {
+            crate::format::PageType::DICTIONARY_PAGE => Ok(PageMetadata {
                 num_rows: usize::MIN,
                 is_dict: true,
             }),
-            parquet_format::PageType::DataPageV2 => Ok(PageMetadata {
+            crate::format::PageType::DATA_PAGE_V2 => Ok(PageMetadata {
                 num_rows: value.data_page_header_v2.as_ref().unwrap().num_rows as usize,
                 is_dict: false,
             }),

parquet/src/column/writer/mod.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 //! Contains column writer API.
-use parquet_format::{ColumnIndex, OffsetIndex};
+use crate::format::{ColumnIndex, OffsetIndex};
 use std::collections::{BTreeSet, VecDeque};
 
 use crate::basic::{Compression, ConvertedType, Encoding, LogicalType, PageType, Type};
@@ -1089,8 +1089,8 @@ fn compare_greater_byte_array_decimals(a: &[u8], b: &[u8]) -> bool {
 
 #[cfg(test)]
 mod tests {
+    use crate::format::BoundaryOrder;
     use bytes::Bytes;
-    use parquet_format::BoundaryOrder;
     use rand::distributions::uniform::SampleUniform;
     use std::sync::Arc;
 
@@ -2086,7 +2086,7 @@ mod tests {
         // column index
         assert_eq!(2, column_index.null_pages.len());
         assert_eq!(2, offset_index.page_locations.len());
-        assert_eq!(BoundaryOrder::Unordered, column_index.boundary_order);
+        assert_eq!(BoundaryOrder::UNORDERED, column_index.boundary_order);
         for idx in 0..2 {
             assert!(!column_index.null_pages[idx]);
             assert_eq!(0, column_index.null_counts.as_ref().unwrap()[idx]);

parquet/src/file/footer.rs

@@ -17,7 +17,7 @@
 
 use std::{io::Read, sync::Arc};
 
-use parquet_format::{ColumnOrder as TColumnOrder, FileMetaData as TFileMetaData};
+use crate::format::{ColumnOrder as TColumnOrder, FileMetaData as TFileMetaData};
 use thrift::protocol::TCompactInputProtocol;
 
 use crate::basic::ColumnOrder;
@@ -149,8 +149,8 @@ mod tests {
 
     use crate::basic::SortOrder;
     use crate::basic::Type;
+    use crate::format::TypeDefinedOrder;
     use crate::schema::types::Type as SchemaType;
-    use parquet_format::TypeDefinedOrder;
 
     #[test]
     fn test_parse_metadata_size_smaller_than_footer() {

parquet/src/file/metadata.rs

@@ -35,7 +35,7 @@
 
 use std::sync::Arc;
 
-use parquet_format::{
+use crate::format::{
     BoundaryOrder, ColumnChunk, ColumnIndex, ColumnMetaData, OffsetIndex, PageLocation,
     RowGroup,
 };
@@ -122,7 +122,7 @@ impl ParquetMetaData {
     }
 }
 
-pub type KeyValue = parquet_format::KeyValue;
+pub type KeyValue = crate::format::KeyValue;
 
 /// Reference counted pointer for [`FileMetaData`].
 pub type FileMetaDataPtr = Arc<FileMetaData>;
@@ -553,14 +553,14 @@ impl ColumnChunkMetaData {
             return Err(general_err!("Expected to have column metadata"));
         }
         let mut col_metadata: ColumnMetaData = cc.meta_data.unwrap();
-        let column_type = Type::from(col_metadata.type_);
+        let column_type = Type::try_from(col_metadata.type_)?;
         let column_path = ColumnPath::new(col_metadata.path_in_schema);
         let encodings = col_metadata
             .encodings
             .drain(0..)
-            .map(Encoding::from)
-            .collect();
-        let compression = Compression::from(col_metadata.codec);
+            .map(Encoding::try_from)
+            .collect::<Result<_>>()?;
+        let compression = Compression::try_from(col_metadata.codec)?;
         let file_path = cc.file_path;
         let file_offset = cc.file_offset;
         let num_values = col_metadata.num_values;
@@ -573,7 +573,12 @@ impl ColumnChunkMetaData {
         let encoding_stats = col_metadata
             .encoding_stats
             .as_ref()
-            .map(|vec| vec.iter().map(page_encoding_stats::from_thrift).collect());
+            .map(|vec| {
+                vec.iter()
+                    .map(page_encoding_stats::try_from_thrift)
+                    .collect::<Result<_>>()
+            })
+            .transpose()?;
         let bloom_filter_offset = col_metadata.bloom_filter_offset;
         let offset_index_offset = cc.offset_index_offset;
         let offset_index_length = cc.offset_index_length;
@@ -846,7 +851,7 @@ impl ColumnIndexBuilder {
             null_pages: Vec::new(),
             min_values: Vec::new(),
             max_values: Vec::new(),
-            boundary_order: BoundaryOrder::Unordered,
+            boundary_order: BoundaryOrder::UNORDERED,
             null_counts: Vec::new(),
             valid: true,
         }

parquet/src/file/page_encoding_stats.rs

@@ -16,7 +16,8 @@
 // under the License.
 
 use crate::basic::{Encoding, PageType};
-use parquet_format::{
+use crate::errors::Result;
+use crate::format::{
     Encoding as TEncoding, PageEncodingStats as TPageEncodingStats, PageType as TPageType,
 };
 
@@ -32,16 +33,18 @@ pub struct PageEncodingStats {
 }
 
 /// Converts Thrift definition into `PageEncodingStats`.
-pub fn from_thrift(thrift_encoding_stats: &TPageEncodingStats) -> PageEncodingStats {
-    let page_type = PageType::from(thrift_encoding_stats.page_type);
-    let encoding = Encoding::from(thrift_encoding_stats.encoding);
+pub fn try_from_thrift(
+    thrift_encoding_stats: &TPageEncodingStats,
+) -> Result<PageEncodingStats> {
+    let page_type = PageType::try_from(thrift_encoding_stats.page_type)?;
+    let encoding = Encoding::try_from(thrift_encoding_stats.encoding)?;
     let count = thrift_encoding_stats.count;
 
-    PageEncodingStats {
+    Ok(PageEncodingStats {
         page_type,
         encoding,
         count,
-    }
+    })
 }
 
 /// Converts `PageEncodingStats` into Thrift definition.
@@ -70,6 +73,6 @@ mod tests {
             count: 1,
         };
 
-        assert_eq!(from_thrift(&to_thrift(&stats)), stats);
+        assert_eq!(try_from_thrift(&to_thrift(&stats)).unwrap(), stats);
     }
 }

parquet/src/file/page_index/index.rs

@@ -19,8 +19,8 @@ use crate::basic::Type;
 use crate::data_type::private::ParquetValueType;
 use crate::data_type::Int96;
 use crate::errors::ParquetError;
+use crate::format::{BoundaryOrder, ColumnIndex};
 use crate::util::bit_util::from_le_slice;
-use parquet_format::{BoundaryOrder, ColumnIndex};
 use std::fmt::Debug;
 
 /// The statistics in one page

parquet/src/file/page_index/index_reader.rs

@@ -21,7 +21,7 @@ use crate::errors::ParquetError;
 use crate::file::metadata::ColumnChunkMetaData;
 use crate::file::page_index::index::{BooleanIndex, ByteArrayIndex, Index, NativeIndex};
 use crate::file::reader::ChunkReader;
-use parquet_format::{ColumnIndex, OffsetIndex, PageLocation};
+use crate::format::{ColumnIndex, OffsetIndex, PageLocation};
 use std::io::{Cursor, Read};
 use thrift::protocol::TCompactInputProtocol;
 

parquet/src/file/page_index/range.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 use crate::errors::ParquetError;
-use parquet_format::PageLocation;
+use crate::format::PageLocation;
 use std::cmp::Ordering;
 use std::collections::VecDeque;
 use std::ops::RangeInclusive;
@@ -284,7 +284,7 @@ mod tests {
     use crate::basic::Type::INT32;
     use crate::file::page_index::index::{NativeIndex, PageIndex};
     use crate::file::page_index::range::{compute_row_ranges, Range, RowRanges};
-    use parquet_format::{BoundaryOrder, PageLocation};
+    use crate::format::{BoundaryOrder, PageLocation};
 
     #[test]
     fn test_binary_search_overlap() {
@@ -445,7 +445,7 @@ mod tests {
                     null_count: Some(0),
                 },
             ],
-            boundary_order: BoundaryOrder::Ascending,
+            boundary_order: BoundaryOrder::ASCENDING,
         };
         let locations = &[
             PageLocation {