diff --git a/.github/workflows/arrow.yml b/.github/workflows/arrow.yml index 466f0b12ec3..e0961daf1d6 100644 --- a/.github/workflows/arrow.yml +++ b/.github/workflows/arrow.yml @@ -30,6 +30,7 @@ on: - arrow-buffer/** - arrow-data/** - arrow-schema/** + - arrow-integration-test/** - .github/** jobs: @@ -60,6 +61,8 @@ jobs: run: cargo test -p arrow-schema --all-features - name: Test arrow-array with all features run: cargo test -p arrow-array --all-features + - name: Test arrow-integration-test with all features + run: cargo test -p arrow-integration-test --all-features - name: Test arrow run: cargo test -p arrow - name: Test --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict,dyn_arith_dict diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 57dc1948276..214a11d5ec8 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -53,5 +53,5 @@ jobs: - name: Prettier check run: | # if you encounter error, run the command below and commit the changes - npx prettier@2.3.2 --write {arrow,arrow-flight,dev,integration-testing,parquet}/**/*.md README.md CODE_OF_CONDUCT.md CONTRIBUTING.md + npx prettier@2.3.2 --write {arrow,arrow-flight,dev,arrow-integration-testing,parquet}/**/*.md README.md CODE_OF_CONDUCT.md CONTRIBUTING.md git diff --exit-code diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0f183990ed4..bdf576af98d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -30,7 +30,8 @@ on: - arrow-data/** - arrow-schema/** - arrow-pyarrow-integration-testing/** - - integration-testing/** + - arrow-integration-test/** + - arrow-integration-testing/** - .github/** jobs: diff --git a/Cargo.toml b/Cargo.toml index 28517265b3c..e57de7711d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,14 +19,15 @@ members = [ "arrow", "arrow-array", - "arrow-data", - "arrow-schema", "arrow-buffer", + "arrow-data", "arrow-flight", + "arrow-integration-test", + "arrow-integration-testing", + "arrow-schema", "parquet", "parquet_derive", "parquet_derive_test", - "integration-testing", "object_store", ] # Enable the version 2 feature resolver, which avoids unifying features for targets that are not being built diff --git a/arrow-integration-test/Cargo.toml b/arrow-integration-test/Cargo.toml new file mode 100644 index 00000000000..be54cc4d623 --- /dev/null +++ b/arrow-integration-test/Cargo.toml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "arrow-integration-test" +version = "24.0.0" +description = "Support for the Apache Arrow JSON test data format" +homepage = "https://github.com/apache/arrow-rs" +repository = "https://github.com/apache/arrow-rs" +authors = ["Apache Arrow "] +license = "Apache-2.0" +keywords = ["arrow"] +include = [ + "benches/*.rs", + "src/**/*.rs", + "Cargo.toml", +] +edition = "2021" +rust-version = "1.62" + +[lib] +name = "arrow_integration_test" +path = "src/lib.rs" +bench = false + +[dependencies] +arrow = { version = "24.0.0", path = "../arrow", default-features = false } +arrow-buffer = { version = "24.0.0", path = "../arrow-buffer" } +hex = { version = "0.4", default-features = false, features = ["std"] } +serde = { version = "1.0", default-features = false, features = ["rc", "derive"] } +serde_json = { version = "1.0", default-features = false, features = ["std"] } +num = { version = "0.4", default-features = false, features = ["std"] } + +[build-dependencies] diff --git a/integration-testing/data/integration.json b/arrow-integration-test/data/integration.json similarity index 100% rename from integration-testing/data/integration.json rename to arrow-integration-test/data/integration.json diff --git a/integration-testing/src/util/datatype.rs b/arrow-integration-test/src/datatype.rs similarity index 100% rename from integration-testing/src/util/datatype.rs rename to arrow-integration-test/src/datatype.rs diff --git a/integration-testing/src/util/field.rs b/arrow-integration-test/src/field.rs similarity index 99% rename from integration-testing/src/util/field.rs rename to arrow-integration-test/src/field.rs index a2becc004d1..9b1a8f5f9ba 100644 --- a/integration-testing/src/util/field.rs +++ b/arrow-integration-test/src/field.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::util::datatype::{data_type_from_json, data_type_to_json}; +use crate::{data_type_from_json, data_type_to_json}; use arrow::datatypes::{DataType, Field}; use arrow::error::{ArrowError, Result}; use std::collections::BTreeMap; diff --git a/integration-testing/src/util/mod.rs b/arrow-integration-test/src/lib.rs similarity index 99% rename from integration-testing/src/util/mod.rs rename to arrow-integration-test/src/lib.rs index 72ecfaa00f0..cf7024dc026 100644 --- a/integration-testing/src/util/mod.rs +++ b/arrow-integration-test/src/lib.rs @@ -15,9 +15,11 @@ // specific language governing permissions and limitations // under the License. -//! Utils for JSON integration testing +//! Support for the [Apache Arrow JSON test data format](https://github.com/apache/arrow/blob/master/docs/source/format/Integration.rst#json-test-data-format) //! //! These utilities define structs that read the integration JSON format for integration testing purposes. +//! +//! This is not a canonical format, but provides a human-readable way of verifying language implementations use hex::decode; use num::BigInt; @@ -40,8 +42,8 @@ mod datatype; mod field; mod schema; -use crate::util::datatype::data_type_to_json; -use crate::util::field::field_from_json; +pub use datatype::*; +pub use field::*; pub use schema::*; /// A struct that represents an Arrow file with a schema and record batches diff --git a/integration-testing/src/util/schema.rs b/arrow-integration-test/src/schema.rs similarity index 99% rename from integration-testing/src/util/schema.rs rename to arrow-integration-test/src/schema.rs index 7e3475e6f46..8147589390a 100644 --- a/integration-testing/src/util/schema.rs +++ b/arrow-integration-test/src/schema.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::util::field::{field_from_json, field_to_json}; +use crate::{field_from_json, field_to_json}; use arrow::datatypes::Schema; use arrow::error::{ArrowError, Result}; use std::collections::HashMap; diff --git a/integration-testing/Cargo.toml b/arrow-integration-testing/Cargo.toml similarity index 96% rename from integration-testing/Cargo.toml rename to arrow-integration-testing/Cargo.toml index 7b28f399f24..0f43447de4c 100644 --- a/integration-testing/Cargo.toml +++ b/arrow-integration-testing/Cargo.toml @@ -34,6 +34,7 @@ logging = ["tracing-subscriber"] arrow = { path = "../arrow", default-features = false, features = ["test_utils", "ipc", "ipc_compression", "json"] } arrow-flight = { path = "../arrow-flight", default-features = false } arrow-buffer = { path = "../arrow-buffer", default-features = false } +arrow-integration-test = { path = "../arrow-integration-test", default-features = false } async-trait = { version = "0.1.41", default-features = false } clap = { version = "4", default-features = false, features = ["std", "derive", "help", "error-context", "usage"] } futures = { version = "0.3", default-features = false } diff --git a/integration-testing/README.md b/arrow-integration-testing/README.md similarity index 100% rename from integration-testing/README.md rename to arrow-integration-testing/README.md diff --git a/integration-testing/src/bin/arrow-file-to-stream.rs b/arrow-integration-testing/src/bin/arrow-file-to-stream.rs similarity index 100% rename from integration-testing/src/bin/arrow-file-to-stream.rs rename to arrow-integration-testing/src/bin/arrow-file-to-stream.rs diff --git a/integration-testing/src/bin/arrow-json-integration-test.rs b/arrow-integration-testing/src/bin/arrow-json-integration-test.rs similarity index 98% rename from integration-testing/src/bin/arrow-json-integration-test.rs rename to arrow-integration-testing/src/bin/arrow-json-integration-test.rs index b84680f6f4b..5eb443b08a8 100644 --- a/integration-testing/src/bin/arrow-json-integration-test.rs +++ b/arrow-integration-testing/src/bin/arrow-json-integration-test.rs @@ -20,7 +20,8 @@ use arrow::datatypes::{DataType, Field}; use arrow::error::{ArrowError, Result}; use arrow::ipc::reader::FileReader; use arrow::ipc::writer::FileWriter; -use arrow_integration_testing::{read_json_file, util::*}; +use arrow_integration_test::*; +use arrow_integration_testing::read_json_file; use clap::Parser; use std::fs::File; diff --git a/integration-testing/src/bin/arrow-stream-to-file.rs b/arrow-integration-testing/src/bin/arrow-stream-to-file.rs similarity index 100% rename from integration-testing/src/bin/arrow-stream-to-file.rs rename to arrow-integration-testing/src/bin/arrow-stream-to-file.rs diff --git a/integration-testing/src/bin/flight-test-integration-client.rs b/arrow-integration-testing/src/bin/flight-test-integration-client.rs similarity index 100% rename from integration-testing/src/bin/flight-test-integration-client.rs rename to arrow-integration-testing/src/bin/flight-test-integration-client.rs diff --git a/integration-testing/src/bin/flight-test-integration-server.rs b/arrow-integration-testing/src/bin/flight-test-integration-server.rs similarity index 100% rename from integration-testing/src/bin/flight-test-integration-server.rs rename to arrow-integration-testing/src/bin/flight-test-integration-server.rs diff --git a/integration-testing/src/flight_client_scenarios.rs b/arrow-integration-testing/src/flight_client_scenarios.rs similarity index 100% rename from integration-testing/src/flight_client_scenarios.rs rename to arrow-integration-testing/src/flight_client_scenarios.rs diff --git a/integration-testing/src/flight_client_scenarios/auth_basic_proto.rs b/arrow-integration-testing/src/flight_client_scenarios/auth_basic_proto.rs similarity index 100% rename from integration-testing/src/flight_client_scenarios/auth_basic_proto.rs rename to arrow-integration-testing/src/flight_client_scenarios/auth_basic_proto.rs diff --git a/integration-testing/src/flight_client_scenarios/integration_test.rs b/arrow-integration-testing/src/flight_client_scenarios/integration_test.rs similarity index 100% rename from integration-testing/src/flight_client_scenarios/integration_test.rs rename to arrow-integration-testing/src/flight_client_scenarios/integration_test.rs diff --git a/integration-testing/src/flight_client_scenarios/middleware.rs b/arrow-integration-testing/src/flight_client_scenarios/middleware.rs similarity index 100% rename from integration-testing/src/flight_client_scenarios/middleware.rs rename to arrow-integration-testing/src/flight_client_scenarios/middleware.rs diff --git a/integration-testing/src/flight_server_scenarios.rs b/arrow-integration-testing/src/flight_server_scenarios.rs similarity index 100% rename from integration-testing/src/flight_server_scenarios.rs rename to arrow-integration-testing/src/flight_server_scenarios.rs diff --git a/integration-testing/src/flight_server_scenarios/auth_basic_proto.rs b/arrow-integration-testing/src/flight_server_scenarios/auth_basic_proto.rs similarity index 100% rename from integration-testing/src/flight_server_scenarios/auth_basic_proto.rs rename to arrow-integration-testing/src/flight_server_scenarios/auth_basic_proto.rs diff --git a/integration-testing/src/flight_server_scenarios/integration_test.rs b/arrow-integration-testing/src/flight_server_scenarios/integration_test.rs similarity index 100% rename from integration-testing/src/flight_server_scenarios/integration_test.rs rename to arrow-integration-testing/src/flight_server_scenarios/integration_test.rs diff --git a/integration-testing/src/flight_server_scenarios/middleware.rs b/arrow-integration-testing/src/flight_server_scenarios/middleware.rs similarity index 100% rename from integration-testing/src/flight_server_scenarios/middleware.rs rename to arrow-integration-testing/src/flight_server_scenarios/middleware.rs diff --git a/integration-testing/src/lib.rs b/arrow-integration-testing/src/lib.rs similarity index 99% rename from integration-testing/src/lib.rs rename to arrow-integration-testing/src/lib.rs index 2345f1967f2..2edd0ed2838 100644 --- a/integration-testing/src/lib.rs +++ b/arrow-integration-testing/src/lib.rs @@ -19,12 +19,11 @@ use serde_json::Value; -use util::*; - use arrow::datatypes::Schema; use arrow::error::Result; use arrow::record_batch::RecordBatch; use arrow::util::test_util::arrow_test_data; +use arrow_integration_test::*; use std::collections::HashMap; use std::fs::File; use std::io::BufReader; @@ -36,7 +35,6 @@ pub const AUTH_PASSWORD: &str = "flight"; pub mod flight_client_scenarios; pub mod flight_server_scenarios; -pub mod util; pub struct ArrowFile { pub schema: Schema, diff --git a/integration-testing/tests/ipc_reader.rs b/arrow-integration-testing/tests/ipc_reader.rs similarity index 100% rename from integration-testing/tests/ipc_reader.rs rename to arrow-integration-testing/tests/ipc_reader.rs diff --git a/integration-testing/tests/ipc_writer.rs b/arrow-integration-testing/tests/ipc_writer.rs similarity index 100% rename from integration-testing/tests/ipc_writer.rs rename to arrow-integration-testing/tests/ipc_writer.rs diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 609a5851cad..bafee11edb7 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -4,7 +4,7 @@ target/* dev/release/rat_exclude_files.txt arrow/test/data/* arrow/test/dependency/* -integration-testing/data/* +arrow-integration-test/data/* parquet_derive/test/dependency/* .gitattributes **.gitignore