From 430de1a9c40ef91a0b30c90ec35acccebc257d2d Mon Sep 17 00:00:00 2001 From: Nicolae Vartolomei Date: Wed, 17 Aug 2022 15:31:16 +0100 Subject: [PATCH] parquet-read: add support to read parquet data from stdin --- parquet/src/bin/parquet-read.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/parquet/src/bin/parquet-read.rs b/parquet/src/bin/parquet-read.rs index 927d96f8cde..733e56173aa 100644 --- a/parquet/src/bin/parquet-read.rs +++ b/parquet/src/bin/parquet-read.rs @@ -41,12 +41,13 @@ extern crate parquet; use clap::Parser; use parquet::file::reader::{FileReader, SerializedFileReader}; use parquet::record::Row; +use std::io::{self, Read}; use std::{fs::File, path::Path}; #[derive(Debug, Parser)] #[clap(author, version, about("Binary file to read data from a Parquet file"), long_about = None)] struct Args { - #[clap(short, long, help("Path to a parquet file"))] + #[clap(short, long, help("Path to a parquet file, or - for stdin"))] file_name: String, #[clap( short, @@ -66,10 +67,20 @@ fn main() { let num_records = args.num_records; let json = args.json; - let path = Path::new(&filename); - let file = File::open(&path).expect("Unable to open file"); - let parquet_reader = - SerializedFileReader::new(file).expect("Failed to create reader"); + let parquet_reader: Box = if filename == "-" { + let mut buf = Vec::new(); + io::stdin() + .read_to_end(&mut buf) + .expect("Failed to read stdin into a buffer"); + Box::new( + SerializedFileReader::new(bytes::Bytes::from(buf)) + .expect("Failed to create reader"), + ) + } else { + let path = Path::new(&filename); + let file = File::open(&path).expect("Unable to open file"); + Box::new(SerializedFileReader::new(file).expect("Failed to create reader")) + }; // Use full schema as projected schema let mut iter = parquet_reader