diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml index a4d4d392191..98ee3b80717 100644 --- a/.github/actions/setup-builder/action.yaml +++ b/.github/actions/setup-builder/action.yaml @@ -20,8 +20,12 @@ description: 'Prepare Rust Build Environment' inputs: rust-version: description: 'version of rust to install (e.g. stable)' - required: true + required: false default: 'stable' + target: + description: 'target architecture(s)' + required: false + default: 'x86_64-unknown-linux-gnu' runs: using: "composite" steps: @@ -51,7 +55,11 @@ runs: shell: bash run: | echo "Installing ${{ inputs.rust-version }}" - rustup toolchain install ${{ inputs.rust-version }} + rustup toolchain install ${{ inputs.rust-version }} --target ${{ inputs.target }} rustup default ${{ inputs.rust-version }} - rustup target add wasm32-unknown-unknown - echo "CARGO_TARGET_DIR=/github/home/target" >> $GITHUB_ENV + - name: Disable debuginfo generation + shell: bash + run: echo "RUSTFLAGS=-C debuginfo=1" >> $GITHUB_ENV + - name: Enable backtraces + shell: bash + run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV diff --git a/.github/workflows/arrow.yml b/.github/workflows/arrow.yml index 3c73f9d5c7c..200faf20839 100644 --- a/.github/workflows/arrow.yml +++ b/.github/workflows/arrow.yml @@ -42,18 +42,12 @@ jobs: runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: submodules: true - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - with: - rust-version: stable - name: Test arrow-buffer with all features run: cargo test -p arrow-buffer --all-features - name: Test arrow-data with all features @@ -66,11 +60,10 @@ jobs: run: cargo test -p arrow-select --all-features - name: Test arrow-integration-test with all features run: cargo test -p arrow-integration-test --all-features - - name: Test arrow + - name: Test arrow with default features run: cargo test -p arrow - - name: Test --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict,dyn_arith_dict - run: | - cargo test -p arrow --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict,dyn_arith_dict + - name: Test arrow with all features apart from simd + run: cargo test -p arrow --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict,dyn_arith_dict,chrono-tz - name: Run examples run: | # Test arrow examples @@ -87,39 +80,26 @@ jobs: runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: submodules: true - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - with: - rust-version: stable - name: Check compilation - run: | - cargo check -p arrow + run: cargo check -p arrow - name: Check compilation --no-default-features - run: | - cargo check -p arrow --no-default-features + run: cargo check -p arrow --no-default-features - name: Check compilation --all-targets - run: | - cargo check -p arrow --all-targets + run: cargo check -p arrow --all-targets - name: Check compilation --no-default-features --all-targets - run: | - cargo check -p arrow --no-default-features --all-targets + run: cargo check -p arrow --no-default-features --all-targets - name: Check compilation --no-default-features --all-targets --features test_utils - run: | - cargo check -p arrow --no-default-features --all-targets --features test_utils + run: cargo check -p arrow --no-default-features --all-targets --features test_utils - name: Check compilation --no-default-features --all-targets --features ffi - run: | - cargo check -p arrow --no-default-features --all-targets --features ffi + run: cargo check -p arrow --no-default-features --all-targets --features ffi - name: Check compilation --no-default-features --all-targets --features chrono-tz - run: | - cargo check -p arrow --no-default-features --all-targets --features chrono-tz + run: cargo check -p arrow --no-default-features --all-targets --features chrono-tz # test the --features "simd" of the arrow crate. This requires nightly Rust. linux-test-simd: @@ -127,10 +107,6 @@ jobs: runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: @@ -140,14 +116,11 @@ jobs: with: rust-version: nightly - name: Run tests --features "simd" - run: | - cargo test -p arrow --features "simd" + run: cargo test -p arrow --features "simd" - name: Check compilation --features "simd" - run: | - cargo check -p arrow --features simd + run: cargo check -p arrow --features simd - name: Check compilation --features simd --all-targets - run: | - cargo check -p arrow --features simd --all-targets + run: cargo check -p arrow --features simd --all-targets # test the arrow crate builds against wasm32 in stable rust @@ -156,30 +129,18 @@ jobs: runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: submodules: true - - name: Cache Cargo - uses: actions/cache@v3 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder with: - path: /github/home/.cargo - key: cargo-wasm32-cache3- - - name: Setup Rust toolchain for WASM - run: | - rustup toolchain install nightly - rustup override set nightly - rustup target add wasm32-unknown-unknown - rustup target add wasm32-wasi - - name: Build - run: | - cd arrow - cargo build --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-unknown-unknown - cargo build --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-wasi + target: wasm32-unknown-unknown,wasm32-wasi + - name: Build wasm32-unknown-unknown + run: cargo build -p arrow --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-unknown-unknown + - name: Build wasm32-wasi + run: cargo build -p arrow --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-wasi clippy: name: Clippy @@ -190,21 +151,17 @@ jobs: - uses: actions/checkout@v3 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - with: - rust-version: stable - name: Setup Clippy - run: | - rustup component add clippy + run: rustup component add clippy - name: Clippy arrow-buffer with all features - run: cargo clippy -p arrow-buffer --all-targets --all-features + run: cargo clippy -p arrow-buffer --all-targets --all-features -- -D warnings - name: Clippy arrow-data with all features - run: cargo clippy -p arrow-data --all-targets --all-features + run: cargo clippy -p arrow-data --all-targets --all-features -- -D warnings - name: Clippy arrow-schema with all features - run: cargo clippy -p arrow-schema --all-targets --all-features + run: cargo clippy -p arrow-schema --all-targets --all-features -- -D warnings - name: Clippy arrow-array with all features - run: cargo clippy -p arrow-array --all-targets --all-features + run: cargo clippy -p arrow-array --all-targets --all-features -- -D warnings - name: Clippy arrow-select with all features - run: cargo clippy -p arrow-select --all-targets --all-features + run: cargo clippy -p arrow-select --all-targets --all-features -- -D warnings - name: Clippy arrow - run: | - cargo clippy -p arrow --features=prettyprint,csv,ipc,test_utils,ffi,ipc_compression,dyn_cmp_dict,dyn_arith_dict --all-targets -- -D warnings + run: cargo clippy -p arrow --features=prettyprint,csv,ipc,test_utils,ffi,ipc_compression,dyn_cmp_dict,dyn_arith_dict --all-targets -- -D warnings diff --git a/.github/workflows/arrow_flight.yml b/.github/workflows/arrow_flight.yml index d40c9b6ecee..548caeb2ab7 100644 --- a/.github/workflows/arrow_flight.yml +++ b/.github/workflows/arrow_flight.yml @@ -43,18 +43,12 @@ jobs: runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: submodules: true - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - with: - rust-version: stable - name: Test run: | cargo test -p arrow-flight @@ -73,11 +67,7 @@ jobs: - uses: actions/checkout@v3 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - with: - rust-version: stable - name: Setup Clippy - run: | - rustup component add clippy + run: rustup component add clippy - name: Run clippy - run: | - cargo clippy -p arrow-flight --all-features -- -D warnings + run: cargo clippy -p arrow-flight --all-features -- -D warnings diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5e82d76febe..e780226b6e2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -51,5 +51,4 @@ jobs: with: rust-version: ${{ matrix.rust }} - name: Run cargo doc - run: | - cargo doc --document-private-items --no-deps --workspace --all-features + run: cargo doc --document-private-items --no-deps --workspace --all-features diff --git a/.github/workflows/miri.yaml b/.github/workflows/miri.yaml index 92d6f2af2a9..435582347e4 100644 --- a/.github/workflows/miri.yaml +++ b/.github/workflows/miri.yaml @@ -49,5 +49,4 @@ jobs: env: RUST_BACKTRACE: full RUST_LOG: "trace" - run: | - bash .github/workflows/miri.sh + run: bash .github/workflows/miri.sh diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index 2afcb434437..f8f7cf12171 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -37,11 +37,10 @@ jobs: image: amd64/rust steps: - uses: actions/checkout@v3 - - name: Setup Rust toolchain with clippy - run: | - rustup toolchain install stable - rustup default stable - rustup component add clippy + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + - name: Setup Clippy + run: rustup component add clippy # Run different tests for the library on its own as well as # all targets to ensure that it still works in the absence of # features that might be enabled by dev-dependencies of other @@ -68,12 +67,6 @@ jobs: name: Emulator Tests runs-on: ubuntu-latest env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - # https://github.com/rust-lang/cargo/issues/10280 - CARGO_NET_GIT_FETCH_WITH_CLI: "true" - RUST_BACKTRACE: "1" # Run integration tests TEST_INTEGRATION: 1 EC2_METADATA_ENDPOINT: http://localhost:1338 @@ -111,9 +104,7 @@ jobs: az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;' - name: Setup Rust toolchain - run: | - rustup toolchain install stable - rustup default stable + uses: ./.github/actions/setup-builder - name: Run object_store tests env: @@ -121,6 +112,23 @@ jobs: OBJECT_STORE_AWS_ACCESS_KEY_ID: test OBJECT_STORE_AWS_SECRET_ACCESS_KEY: test OBJECT_STORE_AWS_ENDPOINT: http://localhost:4566 - run: | - # run tests - cargo test -p object_store --features=aws,azure,gcp + run: cargo test -p object_store --features=aws,azure,gcp + + # test the object_store crate builds against wasm32 in stable rust + wasm32-build: + name: Build wasm32 + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + target: wasm32-unknown-unknown,wasm32-wasi + - name: Build wasm32-unknown-unknown + run: cargo build -p object_store --target wasm32-unknown-unknown + - name: Build wasm32-wasi + run: cargo build -p object_store --target wasm32-wasi \ No newline at end of file diff --git a/.github/workflows/parquet.yml b/.github/workflows/parquet.yml index 550b590737a..7ccb6f71d56 100644 --- a/.github/workflows/parquet.yml +++ b/.github/workflows/parquet.yml @@ -43,24 +43,16 @@ jobs: runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: submodules: true - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - with: - rust-version: stable - name: Test - run: | - cargo test -p parquet + run: cargo test -p parquet - name: Test --all-features - run: | - cargo test -p parquet --all-features + run: cargo test -p parquet --all-features # test compilation @@ -79,8 +71,6 @@ jobs: submodules: true - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - with: - rust-version: stable # Run different tests for the library on its own as well as # all targets to ensure that it still works in the absence of @@ -93,35 +83,42 @@ jobs: # 3. compiles with just arrow feature # 3. compiles with all features - name: Check compilation - run: | - cargo check -p parquet + run: cargo check -p parquet - name: Check compilation --no-default-features - run: | - cargo check -p parquet --no-default-features + run: cargo check -p parquet --no-default-features - name: Check compilation --no-default-features --features arrow - run: | - cargo check -p parquet --no-default-features --features arrow + run: cargo check -p parquet --no-default-features --features arrow - name: Check compilation --no-default-features --all-features - run: | - cargo check -p parquet --all-features + run: cargo check -p parquet --all-features - name: Check compilation --all-targets - run: | - cargo check -p parquet --all-targets + run: cargo check -p parquet --all-targets - name: Check compilation --all-targets --no-default-features - run: | - cargo check -p parquet --all-targets --no-default-features + run: cargo check -p parquet --all-targets --no-default-features - name: Check compilation --all-targets --no-default-features --features arrow - run: | - cargo check -p parquet --all-targets --no-default-features --features arrow + run: cargo check -p parquet --all-targets --no-default-features --features arrow - name: Check compilation --all-targets --all-features - run: | - cargo check -p parquet --all-targets --all-features + run: cargo check -p parquet --all-targets --all-features - name: Check compilation --all-targets --no-default-features --features json - run: | - cargo check -p parquet --all-targets --no-default-features --features json - - name: Check compilation wasm32-unknown-unknown - run: | - cargo check -p parquet --no-default-features --features cli,snap,flate2,brotli --target wasm32-unknown-unknown + run: cargo check -p parquet --all-targets --no-default-features --features json + + # test the parquet crate builds against wasm32 in stable rust + wasm32-build: + name: Build wasm32 + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + target: wasm32-unknown-unknown,wasm32-wasi + - name: Build wasm32-unknown-unknown + run: cargo build -p parquet --no-default-features --features cli,snap,flate2,brotli --target wasm32-unknown-unknown + - name: Build wasm32-wasi + run: cargo build -p parquet --no-default-features --features cli,snap,flate2,brotli --target wasm32-wasi clippy: name: Clippy @@ -132,11 +129,7 @@ jobs: - uses: actions/checkout@v3 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - with: - rust-version: stable - name: Setup Clippy - run: | - rustup component add clippy + run: rustup component add clippy - name: Run clippy - run: | - cargo clippy -p parquet --all-targets --all-features -- -D warnings + run: cargo clippy -p parquet --all-targets --all-features -- -D warnings diff --git a/.github/workflows/parquet_derive.yml b/.github/workflows/parquet_derive.yml index bd70fc30d1c..f8c4ddc43b7 100644 --- a/.github/workflows/parquet_derive.yml +++ b/.github/workflows/parquet_derive.yml @@ -49,11 +49,8 @@ jobs: submodules: true - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - with: - rust-version: stable - name: Test - run: | - cargo test -p parquet_derive + run: cargo test -p parquet_derive clippy: name: Clippy @@ -64,11 +61,7 @@ jobs: - uses: actions/checkout@v3 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - with: - rust-version: stable - name: Setup Clippy - run: | - rustup component add clippy + run: rustup component add clippy - name: Run clippy - run: | - cargo clippy -p parquet_derive --all-features -- -D warnings + run: cargo clippy -p parquet_derive --all-features -- -D warnings diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index c04d5643b49..f4c98c5abad 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -37,8 +37,7 @@ jobs: with: submodules: true - name: Install protoc with brew - run: | - brew install protobuf + run: brew install protobuf - name: Setup Rust toolchain run: | rustup toolchain install stable --no-self-update @@ -91,10 +90,9 @@ jobs: image: amd64/rust steps: - uses: actions/checkout@v3 - - name: Setup toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + - name: Setup rustfmt + run: rustup component add rustfmt - name: Run run: cargo fmt --all -- --check