Cleanup CI

.github/actions/setup-builder/action.yaml

@@ -20,8 +20,12 @@ description: 'Prepare Rust Build Environment'
 inputs:
   rust-version:
     description: 'version of rust to install (e.g. stable)'
-    required: true
+    required: false
     default: 'stable'
+  target:
+    description: 'target architecture(s)'
+    required: false
+    default: 'x86_64-unknown-linux-gnu'
 runs:
   using: "composite"
   steps:
@@ -51,7 +55,11 @@ runs:
       shell: bash
       run: |
         echo "Installing ${{ inputs.rust-version }}"
-        rustup toolchain install ${{ inputs.rust-version }}
+        rustup toolchain install ${{ inputs.rust-version }} --target ${{ inputs.target }}
         rustup default ${{ inputs.rust-version }}
-        rustup target add wasm32-unknown-unknown
-        echo "CARGO_TARGET_DIR=/github/home/target" >> $GITHUB_ENV
+    - name: Disable debuginfo generation
+      shell: bash
+      run: echo "RUSTFLAGS=-C debuginfo=1" >> $GITHUB_ENV
+    - name: Enable backtraces
+      shell: bash
+      run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV

.github/workflows/arrow.yml

@@ -42,18 +42,12 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: true
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
       - name: Test arrow-buffer with all features
         run: cargo test -p arrow-buffer --all-features
       - name: Test arrow-data with all features
@@ -66,11 +60,10 @@ jobs:
         run: cargo test -p arrow-select --all-features
       - name: Test arrow-integration-test with all features
         run: cargo test -p arrow-integration-test --all-features
-      - name: Test arrow
+      - name: Test arrow with default features
         run: cargo test -p arrow
-      - name: Test --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict,dyn_arith_dict
-        run: |
-          cargo test -p arrow --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict,dyn_arith_dict
+      - name: Test arrow with all features apart from simd
+        run: cargo test -p arrow --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict,dyn_arith_dict,chrono-tz
       - name: Run examples
         run: |
           # Test arrow examples
@@ -87,50 +80,33 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: true
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
       - name: Check compilation
-        run: |
-          cargo check -p arrow
+        run: cargo check -p arrow
       - name: Check compilation --no-default-features
-        run: |
-          cargo check -p arrow --no-default-features
+        run: cargo check -p arrow --no-default-features
       - name: Check compilation --all-targets
-        run: |
-          cargo check -p arrow --all-targets
+        run: cargo check -p arrow --all-targets
       - name: Check compilation --no-default-features --all-targets
-        run: |
-          cargo check -p arrow --no-default-features --all-targets
+        run: cargo check -p arrow --no-default-features --all-targets
       - name: Check compilation --no-default-features --all-targets --features test_utils
-        run: |
-          cargo check -p arrow --no-default-features --all-targets --features test_utils
+        run: cargo check -p arrow --no-default-features --all-targets --features test_utils
       - name: Check compilation --no-default-features --all-targets --features ffi
-        run: |
-          cargo check -p arrow --no-default-features --all-targets --features ffi
+        run: cargo check -p arrow --no-default-features --all-targets --features ffi
       - name: Check compilation --no-default-features --all-targets --features chrono-tz
-        run: |
-          cargo check -p arrow --no-default-features --all-targets --features chrono-tz
+        run: cargo check -p arrow --no-default-features --all-targets --features chrono-tz
 
   # test the --features "simd" of the arrow crate. This requires nightly Rust.
   linux-test-simd:
     name: Test SIMD on AMD64 Rust ${{ matrix.rust }}
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
         with:
@@ -140,14 +116,11 @@ jobs:
         with:
           rust-version: nightly
       - name: Run tests --features "simd"
-        run: |
-          cargo test -p arrow --features "simd"
+        run: cargo test -p arrow --features "simd"
       - name: Check compilation --features "simd"
-        run: |
-          cargo check -p arrow --features simd
+        run: cargo check -p arrow --features simd
       - name: Check compilation --features simd --all-targets
-        run: |
-          cargo check -p arrow --features simd --all-targets
+        run: cargo check -p arrow --features simd --all-targets
 
 
   # test the arrow crate builds against wasm32 in stable rust
@@ -156,30 +129,18 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: true
-      - name: Cache Cargo
-        uses: actions/cache@v3
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
         with:
-          path: /github/home/.cargo
-          key: cargo-wasm32-cache3-
-      - name: Setup Rust toolchain for WASM
-        run: |
-          rustup toolchain install nightly
-          rustup override set nightly
-          rustup target add wasm32-unknown-unknown
-          rustup target add wasm32-wasi
-      - name: Build
-        run: |
-          cd arrow
-          cargo build --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-unknown-unknown
-          cargo build --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-wasi
+          target: wasm32-unknown-unknown,wasm32-wasi
+      - name: Build wasm32-unknown-unknown
+        run: cargo build -p arrow --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-unknown-unknown
+      - name: Build wasm32-wasi
+        run: cargo build -p arrow --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-wasi
 
   clippy:
     name: Clippy
@@ -190,21 +151,17 @@ jobs:
       - uses: actions/checkout@v3
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
       - name: Setup Clippy
-        run: |
-          rustup component add clippy
+        run: rustup component add clippy
       - name: Clippy arrow-buffer with all features
-        run: cargo clippy -p arrow-buffer --all-targets --all-features
+        run: cargo clippy -p arrow-buffer --all-targets --all-features -- -D warnings
       - name: Clippy arrow-data with all features
-        run: cargo clippy -p arrow-data --all-targets --all-features
+        run: cargo clippy -p arrow-data --all-targets --all-features -- -D warnings
       - name: Clippy arrow-schema with all features
-        run: cargo clippy -p arrow-schema --all-targets --all-features
+        run: cargo clippy -p arrow-schema --all-targets --all-features -- -D warnings
       - name: Clippy arrow-array with all features
-        run: cargo clippy -p arrow-array --all-targets --all-features
+        run: cargo clippy -p arrow-array --all-targets --all-features -- -D warnings
       - name: Clippy arrow-select with all features
-        run: cargo clippy -p arrow-select --all-targets --all-features
+        run: cargo clippy -p arrow-select --all-targets --all-features -- -D warnings
       - name: Clippy arrow
-        run: |
-          cargo clippy -p arrow --features=prettyprint,csv,ipc,test_utils,ffi,ipc_compression,dyn_cmp_dict,dyn_arith_dict --all-targets -- -D warnings
+        run: cargo clippy -p arrow --features=prettyprint,csv,ipc,test_utils,ffi,ipc_compression,dyn_cmp_dict,dyn_arith_dict --all-targets -- -D warnings

.github/workflows/arrow_flight.yml

@@ -43,18 +43,12 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
-      env:
-        # Disable full debug symbol generation to speed up CI build and keep memory down
-        # "1" means line tables only, which is useful for panic tracebacks.
-        RUSTFLAGS: "-C debuginfo=1"
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: true
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
       - name: Test
         run: |
           cargo test -p arrow-flight
@@ -73,11 +67,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
-        with:
-          rust-version: stable
       - name: Setup Clippy
-        run: |
-          rustup component add clippy
+        run: rustup component add clippy
       - name: Run clippy
-        run: |
-          cargo clippy -p arrow-flight --all-features -- -D warnings
+        run: cargo clippy -p arrow-flight --all-features -- -D warnings

.github/workflows/docs.yml

@@ -51,5 +51,4 @@ jobs:
         with:
           rust-version: ${{ matrix.rust }}
       - name: Run cargo doc
-        run: |
-          cargo doc --document-private-items --no-deps --workspace --all-features
+        run: cargo doc --document-private-items --no-deps --workspace --all-features

.github/workflows/miri.yaml

@@ -49,5 +49,4 @@ jobs:
         env:
           RUST_BACKTRACE: full
           RUST_LOG: "trace"
-        run: |
-          bash .github/workflows/miri.sh
+        run: bash .github/workflows/miri.sh

.github/workflows/object_store.yml

@@ -37,11 +37,10 @@ jobs:
       image: amd64/rust
     steps:
       - uses: actions/checkout@v3
-      - name: Setup Rust toolchain with clippy
-        run: |
-          rustup toolchain install stable
-          rustup default stable
-          rustup component add clippy
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+      - name: Setup Clippy
+        run: rustup component add clippy
       # Run different tests for the library on its own as well as
       # all targets to ensure that it still works in the absence of
       # features that might be enabled by dev-dependencies of other
@@ -68,12 +67,6 @@ jobs:
     name: Emulator Tests
     runs-on: ubuntu-latest
     env:
-      # Disable full debug symbol generation to speed up CI build and keep memory down
-      # "1" means line tables only, which is useful for panic tracebacks.
-      RUSTFLAGS: "-C debuginfo=1"
-      # https://github.com/rust-lang/cargo/issues/10280
-      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
-      RUST_BACKTRACE: "1"
       # Run integration tests
       TEST_INTEGRATION: 1
       EC2_METADATA_ENDPOINT: http://localhost:1338
@@ -111,16 +104,31 @@ jobs:
           az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;'
 
       - name: Setup Rust toolchain
-        run: |
-          rustup toolchain install stable
-          rustup default stable
+        uses: ./.github/actions/setup-builder
 
       - name: Run object_store tests
         env:
           OBJECT_STORE_AWS_DEFAULT_REGION: "us-east-1"
           OBJECT_STORE_AWS_ACCESS_KEY_ID: test
           OBJECT_STORE_AWS_SECRET_ACCESS_KEY: test
           OBJECT_STORE_AWS_ENDPOINT: http://localhost:4566
-        run: |
-          # run tests
-          cargo test -p object_store --features=aws,azure,gcp
+        run: cargo test -p object_store --features=aws,azure,gcp
+
+  # test the object_store crate builds against wasm32 in stable rust
+  wasm32-build:
+    name: Build wasm32
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          target: wasm32-unknown-unknown,wasm32-wasi
+      - name: Build wasm32-unknown-unknown
+        run: cargo build -p object_store --target wasm32-unknown-unknown
+      - name: Build wasm32-wasi
+        run: cargo build -p object_store --target wasm32-wasi