Split out arrow-integration-test crate (#2868)

* Split out arrow-integration-test crate

* Update RAT

* Format

* Update Markdown format

.github/workflows/arrow.yml

@@ -30,6 +30,7 @@ on:
       - arrow-buffer/**
       - arrow-data/**
       - arrow-schema/**
+      - arrow-integration-test/**
       - .github/**
 
 jobs:
@@ -60,6 +61,8 @@ jobs:
         run: cargo test -p arrow-schema --all-features
       - name: Test arrow-array with all features
         run: cargo test -p arrow-array --all-features
+      - name: Test arrow-integration-test with all features
+        run: cargo test -p arrow-integration-test --all-features
       - name: Test arrow
         run: cargo test -p arrow
       - name: Test --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict,dyn_arith_dict

.github/workflows/dev.yml

@@ -53,5 +53,5 @@ jobs:
       - name: Prettier check
         run: |
           # if you encounter error, run the command below and commit the changes
-          npx prettier@2.3.2 --write {arrow,arrow-flight,dev,integration-testing,parquet}/**/*.md README.md CODE_OF_CONDUCT.md CONTRIBUTING.md
+          npx prettier@2.3.2 --write {arrow,arrow-flight,dev,arrow-integration-testing,parquet}/**/*.md README.md CODE_OF_CONDUCT.md CONTRIBUTING.md
           git diff --exit-code

.github/workflows/integration.yml

@@ -30,7 +30,8 @@ on:
       - arrow-data/**
       - arrow-schema/**
       - arrow-pyarrow-integration-testing/**
-      - integration-testing/**
+      - arrow-integration-test/**
+      - arrow-integration-testing/**
       - .github/**
 
 jobs:

Cargo.toml

@@ -19,14 +19,15 @@
 members = [
         "arrow",
         "arrow-array",
-        "arrow-data",
-        "arrow-schema",
         "arrow-buffer",
+        "arrow-data",
         "arrow-flight",
+        "arrow-integration-test",
+        "arrow-integration-testing",
+        "arrow-schema",
         "parquet",
         "parquet_derive",
         "parquet_derive_test",
-        "integration-testing",
         "object_store",
 ]
 # Enable the version 2 feature resolver, which avoids unifying features for targets that are not being built

arrow-integration-test/Cargo.toml

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "arrow-integration-test"
+version = "24.0.0"
+description = "Support for the Apache Arrow JSON test data format"
+homepage = "https://github.com/apache/arrow-rs"
+repository = "https://github.com/apache/arrow-rs"
+authors = ["Apache Arrow <dev@arrow.apache.org>"]
+license = "Apache-2.0"
+keywords = ["arrow"]
+include = [
+    "benches/*.rs",
+    "src/**/*.rs",
+    "Cargo.toml",
+]
+edition = "2021"
+rust-version = "1.62"
+
+[lib]
+name = "arrow_integration_test"
+path = "src/lib.rs"
+bench = false
+
+[dependencies]
+arrow = { version = "24.0.0", path = "../arrow", default-features = false }
+arrow-buffer = { version = "24.0.0", path = "../arrow-buffer" }
+hex = { version = "0.4", default-features = false, features = ["std"] }
+serde = { version = "1.0", default-features = false, features = ["rc", "derive"] }
+serde_json = { version = "1.0", default-features = false, features = ["std"] }
+num = { version = "0.4", default-features = false, features = ["std"] }
+
+[build-dependencies]

integration-testing/src/util/field.rs → arrow-integration-test/src/field.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::util::datatype::{data_type_from_json, data_type_to_json};
+use crate::{data_type_from_json, data_type_to_json};
 use arrow::datatypes::{DataType, Field};
 use arrow::error::{ArrowError, Result};
 use std::collections::BTreeMap;

integration-testing/src/util/mod.rs → arrow-integration-test/src/lib.rs

@@ -15,9 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//! Utils for JSON integration testing
+//! Support for the [Apache Arrow JSON test data format](https://github.com/apache/arrow/blob/master/docs/source/format/Integration.rst#json-test-data-format)
 //!
 //! These utilities define structs that read the integration JSON format for integration testing purposes.
+//!
+//! This is not a canonical format, but provides a human-readable way of verifying language implementations
 
 use hex::decode;
 use num::BigInt;
@@ -40,8 +42,8 @@ mod datatype;
 mod field;
 mod schema;
 
-use crate::util::datatype::data_type_to_json;
-use crate::util::field::field_from_json;
+pub use datatype::*;
+pub use field::*;
 pub use schema::*;
 
 /// A struct that represents an Arrow file with a schema and record batches

integration-testing/src/util/schema.rs → arrow-integration-test/src/schema.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::util::field::{field_from_json, field_to_json};
+use crate::{field_from_json, field_to_json};
 use arrow::datatypes::Schema;
 use arrow::error::{ArrowError, Result};
 use std::collections::HashMap;

integration-testing/Cargo.toml → arrow-integration-testing/Cargo.toml

@@ -34,6 +34,7 @@ logging = ["tracing-subscriber"]
 arrow = { path = "../arrow", default-features = false, features = ["test_utils", "ipc", "ipc_compression", "json"] }
 arrow-flight = { path = "../arrow-flight", default-features = false }
 arrow-buffer = { path = "../arrow-buffer", default-features = false }
+arrow-integration-test = { path = "../arrow-integration-test", default-features = false }
 async-trait = { version = "0.1.41", default-features = false }
 clap = { version = "4", default-features = false, features = ["std", "derive", "help", "error-context", "usage"] }
 futures = { version = "0.3", default-features = false }

integration-testing/src/bin/arrow-json-integration-test.rs → arrow-integration-testing/src/bin/arrow-json-integration-test.rs

@@ -20,7 +20,8 @@ use arrow::datatypes::{DataType, Field};
 use arrow::error::{ArrowError, Result};
 use arrow::ipc::reader::FileReader;
 use arrow::ipc::writer::FileWriter;
-use arrow_integration_testing::{read_json_file, util::*};
+use arrow_integration_test::*;
+use arrow_integration_testing::read_json_file;
 use clap::Parser;
 use std::fs::File;
 

integration-testing/src/lib.rs → arrow-integration-testing/src/lib.rs

@@ -19,12 +19,11 @@
 
 use serde_json::Value;
 
-use util::*;
-
 use arrow::datatypes::Schema;
 use arrow::error::Result;
 use arrow::record_batch::RecordBatch;
 use arrow::util::test_util::arrow_test_data;
+use arrow_integration_test::*;
 use std::collections::HashMap;
 use std::fs::File;
 use std::io::BufReader;
@@ -36,7 +35,6 @@ pub const AUTH_PASSWORD: &str = "flight";
 
 pub mod flight_client_scenarios;
 pub mod flight_server_scenarios;
-pub mod util;
 
 pub struct ArrowFile {
     pub schema: Schema,

dev/release/rat_exclude_files.txt

@@ -4,7 +4,7 @@ target/*
 dev/release/rat_exclude_files.txt
 arrow/test/data/*
 arrow/test/dependency/*
-integration-testing/data/*
+arrow-integration-test/data/*
 parquet_derive/test/dependency/*
 .gitattributes
 **.gitignore