From 9382e7fdaa4fbf67563bfb866be83f7358b12cbe Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Fri, 2 Sep 2022 06:24:59 +0100 Subject: [PATCH] Use http with fake-gcs --- .github/workflows/arrow.yml | 177 --------------------------- .github/workflows/arrow_flight.yml | 76 ------------ .github/workflows/cancel.yml | 54 -------- .github/workflows/coverage.yml | 63 ---------- .github/workflows/dev.yml | 57 --------- .github/workflows/dev_pr.yml | 44 ------- .github/workflows/docs.yml | 55 --------- .github/workflows/integration.yml | 152 ----------------------- .github/workflows/miri.yaml | 48 -------- .github/workflows/object_store.yml | 75 ++++-------- .github/workflows/parquet.yml | 131 -------------------- .github/workflows/parquet_derive.yml | 74 ----------- .github/workflows/rust.yml | 100 --------------- object_store/src/gcp/mod.rs | 21 ---- 14 files changed, 27 insertions(+), 1100 deletions(-) delete mode 100644 .github/workflows/arrow.yml delete mode 100644 .github/workflows/arrow_flight.yml delete mode 100644 .github/workflows/cancel.yml delete mode 100644 .github/workflows/coverage.yml delete mode 100644 .github/workflows/dev.yml delete mode 100644 .github/workflows/dev_pr.yml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/integration.yml delete mode 100644 .github/workflows/miri.yaml delete mode 100644 .github/workflows/parquet.yml delete mode 100644 .github/workflows/parquet_derive.yml delete mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/arrow.yml b/.github/workflows/arrow.yml deleted file mode 100644 index d34ee3b49b5..00000000000 --- a/.github/workflows/arrow.yml +++ /dev/null @@ -1,177 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# tests for arrow crate -name: arrow - -on: - # always trigger - push: - branches: - - master - pull_request: - paths: - - arrow/** - - .github/** - -jobs: - - # test the crate - linux-test: - name: Test - runs-on: ubuntu-latest - container: - image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Test - run: | - cargo test -p arrow - - name: Test --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict - run: | - cargo test -p arrow --features=force_validate,prettyprint,ipc_compression,ffi,dyn_cmp_dict - - name: Run examples - run: | - # Test arrow examples - cargo run --example builders - cargo run --example dynamic_types - cargo run --example read_csv - cargo run --example read_csv_infer_schema - - name: Run non-archery based integration-tests - run: cargo test -p arrow-integration-testing - - # test compilaton features - linux-features: - name: Check Compilation - runs-on: ubuntu-latest - container: - image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Check compilation - run: | - cargo check -p arrow - - name: Check compilation --no-default-features - run: | - cargo check -p arrow --no-default-features - - name: Check compilation --all-targets - run: | - cargo check -p arrow --all-targets - - name: Check compilation --no-default-features --all-targets - run: | - cargo check -p arrow --no-default-features --all-targets - - name: Check compilation --no-default-features --all-targets --features test_utils - run: | - cargo check -p arrow --no-default-features --all-targets --features test_utils - - # test the --features "simd" of the arrow crate. This requires nightly Rust. - linux-test-simd: - name: Test SIMD on AMD64 Rust ${{ matrix.rust }} - runs-on: ubuntu-latest - container: - image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: nightly - - name: Run tests --features "simd" - run: | - cargo test -p arrow --features "simd" - - name: Check compilation --features "simd" - run: | - cargo check -p arrow --features simd - - name: Check compilation --features simd --all-targets - run: | - cargo check -p arrow --features simd --all-targets - - - # test the arrow crate builds against wasm32 in stable rust - wasm32-build: - name: Build wasm32 - runs-on: ubuntu-latest - container: - image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: /github/home/.cargo - key: cargo-wasm32-cache3- - - name: Setup Rust toolchain for WASM - run: | - rustup toolchain install nightly - rustup override set nightly - rustup target add wasm32-unknown-unknown - rustup target add wasm32-wasi - - name: Build - run: | - cd arrow - cargo build --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-unknown-unknown - cargo build --no-default-features --features=json,csv,ipc,simd,ffi --target wasm32-wasi - - clippy: - name: Clippy - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Setup Clippy - run: | - rustup component add clippy - - name: Run clippy - run: | - cargo clippy -p arrow --features=prettyprint,csv,ipc,test_utils,ffi,ipc_compression,dyn_cmp_dict --all-targets -- -D warnings diff --git a/.github/workflows/arrow_flight.yml b/.github/workflows/arrow_flight.yml deleted file mode 100644 index 86a67ff9a6a..00000000000 --- a/.github/workflows/arrow_flight.yml +++ /dev/null @@ -1,76 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -# tests for arrow_flight crate -name: arrow_flight - - -# trigger for all PRs that touch certain files and changes to master -on: - push: - branches: - - master - pull_request: - paths: - - arrow/** - - arrow-flight/** - - .github/** - -jobs: - # test the crate - linux-test: - name: Test - runs-on: ubuntu-latest - container: - image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Test - run: | - cargo test -p arrow-flight - - name: Test --all-features - run: | - cargo test -p arrow-flight --all-features - - clippy: - name: Clippy - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Setup Clippy - run: | - rustup component add clippy - - name: Run clippy - run: | - cargo clippy -p arrow-flight --all-features -- -D warnings diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml deleted file mode 100644 index a98c8ee5d22..00000000000 --- a/.github/workflows/cancel.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Attempt to cancel stale workflow runs to save github actions runner time -name: cancel - -on: - workflow_run: - # The name of another workflow (whichever one) that always runs on PRs - workflows: ['Dev'] - types: ['requested'] - -jobs: - cancel-stale-workflow-runs: - name: "Cancel stale workflow runs" - runs-on: ubuntu-latest - steps: - # Unfortunately, we need to define a separate cancellation step for - # each workflow where we want to cancel stale runs. - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale Dev runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: dev.yml - skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale Integration runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: integration.yml - skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale Rust runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: rust.yml - skipEventTypes: '["push", "schedule"]' diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index e688428e187..00000000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: coverage - -# Trigger only on pushes to master, not pull requests -on: - push: - branches: - - master - -jobs: - - coverage: - name: Coverage - runs-on: ubuntu-latest - # Note runs outside of a container - # otherwise we get this error: - # Failed to run tests: ASLR disable failed: EPERM: Operation not permitted - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - run: | - rustup toolchain install stable - rustup default stable - - name: Install protobuf compiler in /protoc - run: | - sudo mkdir /protoc - sudo chmod a+rwx /protoc - cd /protoc - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protoc-21.4-linux-x86_64.zip - unzip protoc-21.4-linux-x86_64.zip - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: /home/runner/.cargo - key: cargo-coverage-cache3- - - name: Run coverage - run: | - export PATH=$PATH:/protoc/bin - rustup toolchain install stable - rustup default stable - cargo install --version 0.18.2 cargo-tarpaulin - cargo tarpaulin --all --out Xml - - name: Report coverage - continue-on-error: true - run: bash <(curl -s https://codecov.io/bash) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml deleted file mode 100644 index 57dc1948276..00000000000 --- a/.github/workflows/dev.yml +++ /dev/null @@ -1,57 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: dev - -# trigger for all PRs and changes to master -on: - push: - branches: - - master - pull_request: - -env: - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - -jobs: - - rat: - name: Release Audit Tool (RAT) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Audit licenses - run: ./dev/release/run-rat.sh . - - prettier: - name: Markdown format - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: "14" - - name: Prettier check - run: | - # if you encounter error, run the command below and commit the changes - npx prettier@2.3.2 --write {arrow,arrow-flight,dev,integration-testing,parquet}/**/*.md README.md CODE_OF_CONDUCT.md CONTRIBUTING.md - git diff --exit-code diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml deleted file mode 100644 index 38bb3939009..00000000000 --- a/.github/workflows/dev_pr.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: dev_pr - -# Trigger whenever a PR is changed (title as well as new / changed commits) -on: - pull_request_target: - types: - - opened - - edited - - synchronize - -jobs: - process: - name: Process - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Assign GitHub labels - if: | - github.event_name == 'pull_request_target' && - (github.event.action == 'opened' || - github.event.action == 'synchronize') - uses: actions/labeler@v4.0.1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - configuration-path: .github/workflows/dev_pr/labeler.yml - sync-labels: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 5e82d76febe..00000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: docs - -# trigger for all PRs and changes to master -on: - push: - branches: - - master - pull_request: - -jobs: - - # test doc links still work - docs: - name: Rustdocs are clean - runs-on: ubuntu-latest - strategy: - matrix: - arch: [ amd64 ] - rust: [ nightly ] - container: - image: ${{ matrix.arch }}/rust - env: - RUSTDOCFLAGS: "-Dwarnings" - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Install python dev - run: | - apt update - apt install -y libpython3.9-dev - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{ matrix.rust }} - - name: Run cargo doc - run: | - cargo doc --document-private-items --no-deps --workspace --all-features diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index 10a8e30212a..00000000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,152 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: integration - -# trigger for all PRs that touch certain files and changes to master -on: - push: - branches: - - master - pull_request: - paths: - - arrow/** - - arrow-pyarrow-integration-testing/** - - integration-testing/** - - .github/** - -jobs: - - integration: - name: Archery test With other arrows - runs-on: ubuntu-latest - container: - image: apache/arrow-dev:amd64-conda-integration - env: - ARROW_USE_CCACHE: OFF - ARROW_CPP_EXE_PATH: /build/cpp/debug - BUILD_DOCS_CPP: OFF - # These are necessary because the github runner overrides $HOME - # https://github.com/actions/runner/issues/863 - RUSTUP_HOME: /root/.rustup - CARGO_HOME: /root/.cargo - defaults: - run: - shell: bash - steps: - # This is necessary so that actions/checkout can find git - - name: Export conda path - run: echo "/opt/conda/envs/arrow/bin" >> $GITHUB_PATH - # This is necessary so that Rust can find cargo - - name: Export cargo path - run: echo "/root/.cargo/bin" >> $GITHUB_PATH - - name: Check rustup - run: which rustup - - name: Check cmake - run: which cmake - - name: Checkout Arrow - uses: actions/checkout@v3 - with: - repository: apache/arrow - submodules: true - fetch-depth: 0 - - name: Checkout Arrow Rust - uses: actions/checkout@v3 - with: - path: rust - fetch-depth: 0 - - name: Make build directory - run: mkdir /build - - name: Build Rust - run: conda run --no-capture-output ci/scripts/rust_build.sh $PWD /build - - name: Build C++ - run: conda run --no-capture-output ci/scripts/cpp_build.sh $PWD /build - - name: Build C# - run: conda run --no-capture-output ci/scripts/csharp_build.sh $PWD /build - - name: Build Go - run: conda run --no-capture-output ci/scripts/go_build.sh $PWD - - name: Build Java - run: conda run --no-capture-output ci/scripts/java_build.sh $PWD /build - # Temporarily disable JS https://issues.apache.org/jira/browse/ARROW-17410 - # - name: Build JS - # run: conda run --no-capture-output ci/scripts/js_build.sh $PWD /build - - name: Install archery - run: conda run --no-capture-output pip install -e dev/archery - - name: Run integration tests - run: | - conda run --no-capture-output archery integration \ - --run-flight \ - --with-cpp=1 \ - --with-csharp=1 \ - --with-java=1 \ - --with-js=0 \ - --with-go=1 \ - --with-rust=1 \ - --gold-dirs=testing/data/arrow-ipc-stream/integration/0.14.1 \ - --gold-dirs=testing/data/arrow-ipc-stream/integration/0.17.1 \ - --gold-dirs=testing/data/arrow-ipc-stream/integration/1.0.0-bigendian \ - --gold-dirs=testing/data/arrow-ipc-stream/integration/1.0.0-littleendian \ - --gold-dirs=testing/data/arrow-ipc-stream/integration/2.0.0-compression \ - --gold-dirs=testing/data/arrow-ipc-stream/integration/4.0.0-shareddict - - # test FFI against the C-Data interface exposed by pyarrow - pyarrow-integration-test: - name: Pyarrow C Data Interface - runs-on: ubuntu-latest - strategy: - matrix: - rust: [ stable ] - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - run: | - rustup toolchain install ${{ matrix.rust }} - rustup default ${{ matrix.rust }} - rustup component add rustfmt clippy - - name: Cache Cargo - uses: actions/cache@v3 - with: - path: /home/runner/.cargo - key: cargo-maturin-cache- - - name: Cache Rust dependencies - uses: actions/cache@v3 - with: - path: /home/runner/target - # this key is not equal because maturin uses different compilation flags. - key: ${{ runner.os }}-${{ matrix.arch }}-target-maturin-cache-${{ matrix.rust }}- - - uses: actions/setup-python@v4 - with: - python-version: '3.7' - - name: Upgrade pip and setuptools - run: pip install --upgrade pip setuptools wheel virtualenv - - name: Create virtualenv and install dependencies - run: | - virtualenv venv - source venv/bin/activate - pip install maturin toml pytest pytz pyarrow>=5.0 - - name: Run tests - env: - CARGO_HOME: "/home/runner/.cargo" - CARGO_TARGET_DIR: "/home/runner/target" - run: | - source venv/bin/activate - pushd arrow-pyarrow-integration-testing - maturin develop - pytest -v . - popd diff --git a/.github/workflows/miri.yaml b/.github/workflows/miri.yaml deleted file mode 100644 index b4669bbcccc..00000000000 --- a/.github/workflows/miri.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: miri - -# trigger for all PRs that touch certain files and changes to master -on: - push: - branches: - - master - pull_request: - paths: - - arrow/** - - .github/** - -jobs: - miri-checks: - name: MIRI - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - run: | - rustup toolchain install nightly --component miri - rustup override set nightly - cargo miri setup - - name: Run Miri Checks - env: - RUST_BACKTRACE: full - RUST_LOG: "trace" - run: | - bash .github/workflows/miri.sh diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index 118dbd932a4..c905024185d 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -60,74 +60,53 @@ jobs: run: cargo clippy -p object_store --all-features --all-targets -- -D warnings # test the crate + # This runs outside a container to workaround lack of support for passing arguments + # to service containers - https://github.com/orgs/community/discussions/26688 linux-test: name: Emulator Tests runs-on: ubuntu-latest - services: - fake-gcs: - image: fsouza/fake-gcs-server - ports: - - 4443:4443 - localstack: - image: localstack/localstack:0.14.4 - ports: - - 4566:4566 - ec2-metadata: - image: amazon/amazon-ec2-metadata-mock:v1.9.2 - ports: - - 1338:1338 - env: - # Only allow IMDSv2 - AEMM_IMDSV2: "1" - azurite: - image: mcr.microsoft.com/azure-storage/azurite - ports: - - 10000:10002 - container: - image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - # https://github.com/rust-lang/cargo/issues/10280 - CARGO_NET_GIT_FETCH_WITH_CLI: "true" - RUST_BACKTRACE: "1" - # Run integration tests - TEST_INTEGRATION: 1 - EC2_METADATA_ENDPOINT: http://ec2-metadata:1338 - AZURE_USE_EMULATOR: "1" - AZURITE_BLOB_STORAGE_URL: "http://azurite:10000" - AZURITE_QUEUE_STORAGE_URL: "http://azurite:10001" - GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json" - OBJECT_STORE_BUCKET: test-bucket + env: + # Disable full debug symbol generation to speed up CI build and keep memory down + # "1" means line tables only, which is useful for panic tracebacks. + RUSTFLAGS: "-C debuginfo=1" + # https://github.com/rust-lang/cargo/issues/10280 + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + RUST_BACKTRACE: "1" + # Run integration tests + TEST_INTEGRATION: 1 + EC2_METADATA_ENDPOINT: http://localhost:1338 + AZURE_USE_EMULATOR: "1" + AZURITE_BLOB_STORAGE_URL: "http://localhost:10000" + AZURITE_QUEUE_STORAGE_URL: "http://localhost:10001" + GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json" + OBJECT_STORE_BUCKET: test-bucket steps: - uses: actions/checkout@v3 - name: Configure Fake GCS Server (GCP emulation) run: | - curl --insecure -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "https://fake-gcs:4443/storage/v1/b" - echo '{"gcs_base_url": "https://fake-gcs:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT" + docker run -d -p 4443:4443 fsouza/fake-gcs-server -scheme http + curl -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b" + echo '{"gcs_base_url": "http://fake-gcs:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT" - name: Setup LocalStack (AWS emulation) env: AWS_DEFAULT_REGION: "us-east-1" AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test - AWS_ENDPOINT: http://localstack:4566 + AWS_ENDPOINT: http://localhost:4566 run: | - cd /tmp - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - unzip awscliv2.zip - ./aws/install - aws --endpoint-url=http://localstack:4566 s3 mb s3://test-bucket + docker run -d -p 4566:4566 localstack/localstack:0.14.4 + docker run -d -p 1338:1338 amazon/amazon-ec2-metadata-mock:v1.9.2 --imdsv2 + aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket - name: Configure Azurite (Azure emulation) # the magical connection string is from # https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings run: | - curl -sL https://aka.ms/InstallAzureCLIDeb | bash - az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;' + docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite + az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;' - name: Setup Rust toolchain run: | @@ -139,7 +118,7 @@ jobs: OBJECT_STORE_AWS_DEFAULT_REGION: "us-east-1" OBJECT_STORE_AWS_ACCESS_KEY_ID: test OBJECT_STORE_AWS_SECRET_ACCESS_KEY: test - OBJECT_STORE_AWS_ENDPOINT: http://localstack:4566 + OBJECT_STORE_AWS_ENDPOINT: http://localhost:4566 run: | # run tests cargo test -p object_store --features=aws,azure,gcp diff --git a/.github/workflows/parquet.yml b/.github/workflows/parquet.yml deleted file mode 100644 index 42cb06bb0a8..00000000000 --- a/.github/workflows/parquet.yml +++ /dev/null @@ -1,131 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -# tests for parquet crate -name: "parquet" - - -# trigger for all PRs that touch certain files and changes to master -on: - push: - branches: - - master - pull_request: - paths: - - arrow/** - - parquet/** - - .github/** - -jobs: - # test the crate - linux-test: - name: Test - runs-on: ubuntu-latest - container: - image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Test - run: | - cargo test -p parquet - - name: Test --all-features - run: | - cargo test -p parquet --all-features - - - # test compilation - linux-features: - name: Check Compilation - runs-on: ubuntu-latest - container: - image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - # Run different tests for the library on its own as well as - # all targets to ensure that it still works in the absence of - # features that might be enabled by dev-dependencies of other - # targets. - # - # This for each of (library and all-targets), check - # 1. compiles with default features - # 1. compiles with no default features - # 3. compiles with just arrow feature - # 3. compiles with all features - - name: Check compilation - run: | - cargo check -p parquet - - name: Check compilation --no-default-features - run: | - cargo check -p parquet --no-default-features - - name: Check compilation --no-default-features --features arrow - run: | - cargo check -p parquet --no-default-features --features arrow - - name: Check compilation --no-default-features --all-features - run: | - cargo check -p parquet --all-features - - name: Check compilation --all-targets - run: | - cargo check -p parquet --all-targets - - name: Check compilation --all-targets --no-default-features - run: | - cargo check -p parquet --all-targets --no-default-features - - name: Check compilation --all-targets --no-default-features --features arrow - run: | - cargo check -p parquet --all-targets --no-default-features --features arrow - - name: Check compilation --all-targets --all-features - run: | - cargo check -p parquet --all-targets --all-features - - clippy: - name: Clippy - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Setup Clippy - run: | - rustup component add clippy - - name: Run clippy - run: | - cargo clippy -p parquet --all-targets --all-features -- -D warnings diff --git a/.github/workflows/parquet_derive.yml b/.github/workflows/parquet_derive.yml deleted file mode 100644 index bd70fc30d1c..00000000000 --- a/.github/workflows/parquet_derive.yml +++ /dev/null @@ -1,74 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -# tests for parquet_derive crate -name: parquet_derive - - -# trigger for all PRs that touch certain files and changes to master -on: - push: - branches: - - master - pull_request: - paths: - - parquet/** - - parquet_derive/** - - parquet_derive_test/** - - .github/** - -jobs: - # test the crate - linux-test: - name: Test - runs-on: ubuntu-latest - container: - image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Test - run: | - cargo test -p parquet_derive - - clippy: - name: Clippy - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Setup Clippy - run: | - rustup component add clippy - - name: Run clippy - run: | - cargo clippy -p parquet_derive --all-features -- -D warnings diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index c04d5643b49..00000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,100 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# workspace wide tests -name: rust - -# trigger for all PRs and changes to master -on: - push: - branches: - - master - pull_request: - -jobs: - - # Check workspace wide compile and test with default features for - # mac - macos: - name: Test on Mac - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Install protoc with brew - run: | - brew install protobuf - - name: Setup Rust toolchain - run: | - rustup toolchain install stable --no-self-update - rustup default stable - - name: Run tests - shell: bash - run: | - # do not produce debug symbols to keep memory usage down - export RUSTFLAGS="-C debuginfo=0" - cargo test - - - # Check workspace wide compile and test with default features for - # windows - windows: - name: Test on Windows - runs-on: windows-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Install protobuf compiler in /d/protoc - shell: bash - run: | - mkdir /d/protoc - cd /d/protoc - curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protoc-21.4-win64.zip - unzip protoc-21.4-win64.zip - export PATH=$PATH:/d/protoc/bin - protoc --version - - - name: Setup Rust toolchain - run: | - rustup toolchain install stable --no-self-update - rustup default stable - - name: Run tests - shell: bash - run: | - # do not produce debug symbols to keep memory usage down - export RUSTFLAGS="-C debuginfo=0" - export PATH=$PATH:/d/protoc/bin - cargo test - - - # Run cargo fmt for all crates - lint: - name: Lint (cargo fmt) - runs-on: ubuntu-latest - container: - image: amd64/rust - steps: - - uses: actions/checkout@v3 - - name: Setup toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt - - name: Run - run: cargo fmt --all -- --check diff --git a/object_store/src/gcp/mod.rs b/object_store/src/gcp/mod.rs index 65adf9128a4..0ef4d3564b6 100644 --- a/object_store/src/gcp/mod.rs +++ b/object_store/src/gcp/mod.rs @@ -782,20 +782,6 @@ impl GoogleCloudStorageBuilder { self } - /// Use the specified http [`Client`] (defaults to [`Client::new`]) - /// - /// This allows you to set custom client options such as allowing - /// non secure connections or custom headers. - /// - /// NOTE: Currently only available in `test`s to facilitate - /// testing, to avoid leaking details and preserve our ability to - /// make changes to the implementation. - #[cfg(test)] - pub fn with_client(mut self, client: Client) -> Self { - self.client = Some(client); - self - } - /// Configure a connection to Google Cloud Storage, returning a /// new [`GoogleCloudStorage`] and consuming `self` pub fn build(self) -> Result { @@ -923,13 +909,6 @@ mod test { env::var("GOOGLE_SERVICE_ACCOUNT") .expect("already checked GOOGLE_SERVICE_ACCOUNT") ) - .with_client( - // ignore HTTPS errors in tests so we can use fake-gcs server - Client::builder() - .danger_accept_invalid_certs(true) - .build() - .expect("Error creating http client for testing") - ) } }}; }