Merge remote-tracking branch 'upstream/master' into dictionary-re-encode

.asf.yaml

@@ -38,4 +38,10 @@ github:
         # require branches to be up-to-date before merging
         strict: true
         # don't require any jobs to pass
-        contexts: []
+        contexts: []
+
+# publishes the content of the `asf-site` branch to
+# https://arrow.apache.org/rust/
+publish:
+  whoami: asf-site
+  subdir: rust

.github/actions/setup-builder/action.yaml

@@ -65,3 +65,7 @@ runs:
     - name: Enable backtraces
       shell: bash
       run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
+    - name: Fixup git permissions
+      # https://github.com/actions/checkout/issues/766
+      shell: bash
+      run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

.github/dependabot.yml

@@ -6,10 +6,17 @@ updates:
       interval: daily
     open-pull-requests-limit: 10
     target-branch: master
-    labels: [auto-dependencies]
+    labels: [ auto-dependencies, arrow ]
+  - package-ecosystem: cargo
+    directory: "/object_store"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    target-branch: master
+    labels: [ auto-dependencies, object_store ]
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "daily"
     open-pull-requests-limit: 10
-    labels: [auto-dependencies]
+    labels: [ auto-dependencies ]

.github/workflows/arrow.yml

@@ -80,18 +80,18 @@ jobs:
         run: cargo test -p arrow-json --all-features
       - name: Test arrow-string with all features
         run: cargo test -p arrow-string --all-features
-      - name: Test arrow-ord with all features except SIMD
-        run: cargo test -p arrow-ord --features dyn_cmp_dict
+      - name: Test arrow-ord with all features
+        run: cargo test -p arrow-ord --all-features
       - name: Test arrow-arith with all features except SIMD
-        run: cargo test -p arrow-arith --features dyn_arith_dict
+        run: cargo test -p arrow-arith
       - name: Test arrow-row with all features
         run: cargo test -p arrow-row --all-features
       - name: Test arrow-integration-test with all features
         run: cargo test -p arrow-integration-test --all-features
       - name: Test arrow with default features
         run: cargo test -p arrow
       - name: Test arrow with all features apart from simd
-        run: cargo test -p arrow --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict,dyn_arith_dict,chrono-tz
+        run: cargo test -p arrow --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict,chrono-tz
       - name: Run examples
         run: |
           # Test arrow examples
@@ -145,8 +145,6 @@ jobs:
           rust-version: nightly
       - name: Test arrow-array with SIMD
         run: cargo test -p arrow-array --features simd
-      - name: Test arrow-ord with SIMD
-        run: cargo test -p arrow-ord --features simd
       - name: Test arrow-arith with SIMD
         run: cargo test -p arrow-arith --features simd
       - name: Test arrow with SIMD
@@ -206,14 +204,14 @@ jobs:
         run: cargo clippy -p arrow-json --all-targets --all-features -- -D warnings
       - name: Clippy arrow-string with all features
         run: cargo clippy -p arrow-string --all-targets --all-features -- -D warnings
-      - name: Clippy arrow-ord with all features except SIMD
-        run: cargo clippy -p arrow-ord --all-targets --features dyn_cmp_dict -- -D warnings
+      - name: Clippy arrow-ord with all features
+        run: cargo clippy -p arrow-ord --all-targets --all-features -- -D warnings
       - name: Clippy arrow-arith with all features except SIMD
-        run: cargo clippy -p arrow-arith --all-targets --features dyn_arith_dict -- -D warnings
+        run: cargo clippy -p arrow-arith --all-targets -- -D warnings
       - name: Clippy arrow-row with all features
         run: cargo clippy -p arrow-row --all-targets --all-features -- -D warnings
       - name: Clippy arrow with all features except SIMD
-        run: cargo clippy -p arrow --features=prettyprint,csv,ipc,test_utils,ffi,ipc_compression,dyn_cmp_dict,dyn_arith_dict,chrono-tz --all-targets -- -D warnings
+        run: cargo clippy -p arrow --features=prettyprint,csv,ipc,test_utils,ffi,ipc_compression,dyn_cmp_dict,chrono-tz --all-targets -- -D warnings
       - name: Clippy arrow-integration-test with all features
         run: cargo clippy -p arrow-integration-test --all-targets --all-features -- -D warnings
       - name: Clippy arrow-integration-testing with all features

.github/workflows/arrow_flight.yml

@@ -41,7 +41,6 @@ on:
       - .github/**
 
 jobs:
-  # test the crate
   linux-test:
     name: Test
     runs-on: ubuntu-latest
@@ -60,12 +59,21 @@ jobs:
         run: |
           cargo test -p arrow-flight --all-features
       - name: Test --examples
-        run: |
-          cargo test -p arrow-flight  --features=flight-sql-experimental --examples
-      - name: Test --examples with TLS
         run: |
           cargo test -p arrow-flight  --features=flight-sql-experimental,tls --examples
-      - name: Verify workspace clean
+
+  vendor:
+    name: Verify Vendored Code
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+      - name: Run gen
+        run: ./arrow-flight/regen.sh
+      - name: Verify workspace clean (if this fails, run ./arrow-flight/regen.sh and check in results)
         run: git diff --exit-code
 
   clippy:

.github/workflows/dev_pr.yml

@@ -44,7 +44,7 @@ jobs:
           github.event_name == 'pull_request_target' &&
             (github.event.action == 'opened' ||
              github.event.action == 'synchronize')
-        uses: actions/labeler@4.1.0
+        uses: actions/labeler@v4.3.0
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           configuration-path: .github/workflows/dev_pr/labeler.yml

.github/workflows/dev_pr/labeler.yml

@@ -16,13 +16,19 @@
 # under the License.
 
 arrow:
+  - arrow-arith/**/*
   - arrow-array/**/*
   - arrow-buffer/**/*
   - arrow-cast/**/*
   - arrow-csv/**/*
   - arrow-data/**/*
+  - arrow-flight/**/*
+  - arrow-integration-test/**/*
+  - arrow-integration-testing/**/*
   - arrow-ipc/**/*
   - arrow-json/**/*
+  - arrow-ord/**/*
+  - arrow-row/**/*
   - arrow-schema/**/*
   - arrow-select/**/*
   - arrow-string/**/*

.github/workflows/docs.yml

@@ -41,18 +41,57 @@ jobs:
     container:
       image: ${{ matrix.arch }}/rust
       env:
-        RUSTDOCFLAGS: "-Dwarnings"
+        RUSTDOCFLAGS: "-Dwarnings --enable-index-page -Zunstable-options"
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: true
       - name: Install python dev
         run: |
           apt update
-          apt install -y libpython3.9-dev
+          apt install -y libpython3.11-dev
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
           rust-version: ${{ matrix.rust }}
       - name: Run cargo doc
         run: cargo doc --document-private-items --no-deps --workspace --all-features
+      - name: Fix file permissions
+        shell: sh
+        run: |
+          chmod -c -R +rX "target/doc" |
+          while read line; do
+              echo "::warning title=Invalid file permissions automatically fixed::$line"
+          done
+      - name: Upload artifacts
+        uses: actions/upload-pages-artifact@v2
+        with:
+          name: crate-docs
+          path: target/doc
+
+  deploy:
+    # Only deploy if a push to master
+    if: github.ref_name == 'master' && github.event_name == 'push'
+    needs: docs
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download crate docs
+        uses: actions/download-artifact@v3
+        with:
+          name: crate-docs
+          path: website/build
+      - name: Prepare website
+        run: |
+          tar -xf website/build/artifact.tar -C website/build
+          rm website/build/artifact.tar
+          cp .asf.yaml ./website/build/.asf.yaml
+      - name: Deploy to gh-pages
+        uses: peaceiris/actions-gh-pages@v3.9.3
+        if: github.event_name == 'push' && github.ref_name == 'master'
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: website/build
+          publish_branch: asf-site

.github/workflows/miri.sh

@@ -5,13 +5,14 @@
 # Must be run with nightly rust for example
 # rustup default nightly
 
-
-# stacked borrows checking uses too much memory to run successfully in github actions
-# re-enable if the CI is migrated to something more powerful (https://github.com/apache/arrow-rs/issues/1833)
-# see also https://github.com/rust-lang/miri/issues/1367
-export MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-stacked-borrows"
+export MIRIFLAGS="-Zmiri-disable-isolation"
 cargo miri setup
 cargo clean
 
 echo "Starting Arrow MIRI run..."
-cargo miri test -p arrow -- --skip csv --skip ipc --skip json
+cargo miri test -p arrow-buffer
+cargo miri test -p arrow-data --features ffi
+cargo miri test -p arrow-schema --features ffi
+cargo miri test -p arrow-array
+cargo miri test -p arrow-arith --features simd
+cargo miri test -p arrow-ord

.github/workflows/object_store.yml

@@ -39,6 +39,9 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
+    defaults:
+      run:
+        working-directory: object_store
     steps:
       - uses: actions/checkout@v3
       - name: Setup Rust toolchain
@@ -50,26 +53,45 @@ jobs:
       # features that might be enabled by dev-dependencies of other
       # targets.
       - name: Run clippy with default features
-        run: cargo clippy -p object_store -- -D warnings
+        run: cargo clippy -- -D warnings
       - name: Run clippy with aws feature
-        run: cargo clippy -p object_store --features aws -- -D warnings
-      - name: Run clippy with aws_profile feature
-        run: cargo clippy -p object_store --features aws_profile -- -D warnings
+        run: cargo clippy --features aws -- -D warnings
       - name: Run clippy with gcp feature
-        run: cargo clippy -p object_store --features gcp -- -D warnings
+        run: cargo clippy --features gcp -- -D warnings
       - name: Run clippy with azure feature
-        run: cargo clippy -p object_store --features azure -- -D warnings
+        run: cargo clippy --features azure -- -D warnings
       - name: Run clippy with all features
-        run: cargo clippy -p object_store --all-features -- -D warnings
+        run: cargo clippy --all-features -- -D warnings
       - name: Run clippy with all features and all targets
-        run: cargo clippy -p object_store --all-features --all-targets -- -D warnings
+        run: cargo clippy --all-features --all-targets -- -D warnings
+
+  # test doc links still work
+  #
+  # Note that since object_store is not part of the main workspace,
+  # this needs a separate docs job as it is not covered by
+  # `cargo doc --workspace`
+  docs:
+    name: Rustdocs
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: object_store
+    env:
+      RUSTDOCFLAGS: "-Dwarnings"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run cargo doc
+        run: cargo doc --document-private-items --no-deps --workspace --all-features
 
   # test the crate
   # This runs outside a container to workaround lack of support for passing arguments
   # to service containers - https://github.com/orgs/community/discussions/26688
   linux-test:
     name: Emulator Tests
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: object_store
     env:
       # Disable full debug symbol generation to speed up CI build and keep memory down
       # "1" means line tables only, which is useful for panic tracebacks.
@@ -78,19 +100,27 @@ jobs:
       # Run integration tests
       TEST_INTEGRATION: 1
       EC2_METADATA_ENDPOINT: http://localhost:1338
-      AZURE_USE_EMULATOR: "1"
+      AZURE_CONTAINER_NAME: test-bucket
+      AZURE_STORAGE_USE_EMULATOR: "1"
       AZURITE_BLOB_STORAGE_URL: "http://localhost:10000"
       AZURITE_QUEUE_STORAGE_URL: "http://localhost:10001"
+      AWS_BUCKET: test-bucket
+      AWS_DEFAULT_REGION: "us-east-1"
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: test
+      AWS_ENDPOINT: http://localhost:4566
+      AWS_ALLOW_HTTP: true
       HTTP_URL: "http://localhost:8080"
+      GOOGLE_BUCKET: test-bucket
       GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"
-      OBJECT_STORE_BUCKET: test-bucket
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Configure Fake GCS Server (GCP emulation)
+        # Custom image - see fsouza/fake-gcs-server#1164
         run: |
-          docker run -d -p 4443:4443 fsouza/fake-gcs-server -scheme http
+          docker run -d -p 4443:4443 tustvold/fake-gcs-server -scheme http -backend memory -public-host localhost:4443
           # Give the container a moment to start up prior to configuring it
           sleep 1
           curl -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b"
@@ -100,13 +130,8 @@ jobs:
         run: docker run -d -p 8080:80 rclone/rclone serve webdav /data --addr :80
 
       - name: Setup LocalStack (AWS emulation)
-        env:
-          AWS_DEFAULT_REGION: "us-east-1"
-          AWS_ACCESS_KEY_ID: test
-          AWS_SECRET_ACCESS_KEY: test
-          AWS_ENDPOINT: http://localhost:4566
         run: |
-          docker run -d -p 4566:4566 localstack/localstack:0.14.4
+          docker run -d -p 4566:4566 localstack/localstack:2.0
           docker run -d -p 1338:1338 amazon/amazon-ec2-metadata-mock:v1.9.2 --imdsv2
           aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket
 
@@ -123,19 +148,17 @@ jobs:
           rustup default stable
 
       - name: Run object_store tests
-        env:
-          OBJECT_STORE_AWS_DEFAULT_REGION: "us-east-1"
-          OBJECT_STORE_AWS_ACCESS_KEY_ID: test
-          OBJECT_STORE_AWS_SECRET_ACCESS_KEY: test
-          OBJECT_STORE_AWS_ENDPOINT: http://localhost:4566
-        run: cargo test -p object_store --features=aws,azure,gcp,http
+        run: cargo test --features=aws,azure,gcp,http
 
   # test the object_store crate builds against wasm32 in stable rust
   wasm32-build:
     name: Build wasm32
     runs-on: ubuntu-latest
     container:
       image: amd64/rust
+    defaults:
+      run:
+        working-directory: object_store
     steps:
       - uses: actions/checkout@v3
         with:
@@ -145,6 +168,6 @@ jobs:
         with:
           target: wasm32-unknown-unknown,wasm32-wasi
       - name: Build wasm32-unknown-unknown
-        run: cargo build -p object_store --target wasm32-unknown-unknown
+        run: cargo build --target wasm32-unknown-unknown
       - name: Build wasm32-wasi
-        run: cargo build -p object_store --target wasm32-wasi
+        run: cargo build --target wasm32-wasi

.github/workflows/parquet.yml

@@ -60,6 +60,12 @@ jobs:
         run: cargo test -p parquet
       - name: Test --all-features
         run: cargo test -p parquet --all-features
+      - name: Run examples
+        run: |
+          # Test parquet examples
+          cargo run -p parquet --example read_parquet
+          cargo run -p parquet --example async_read_parquet --features="async"
+          cargo run -p parquet --example read_with_rowgroup --features="async"
 
   # test compilation
   linux-features: