360 lines (345 loc) · 12.2 KB
/
rust.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Rust
on:
# always trigger
push:
pull_request:
jobs:
# build the library, a compilation step used by multiple steps below
linux-build-lib:
name: Build Libraries on AMD64 Rust ${{ matrix.rust }}
runs-on: ubuntu-latest
strategy:
matrix:
arch: [ amd64 ]
rust: [ stable ]
container:
image: ${{ matrix.arch }}/rust
env:
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- uses: actions/checkout@v2
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- name: Build Workspace
run: |
cargo build
# test the crate
linux-test:
name: Test Workspace on AMD64 Rust ${{ matrix.rust }}
needs: [ linux-build-lib ]
runs-on: ubuntu-latest
strategy:
matrix:
arch: [ amd64 ]
rust: [ stable ]
container:
image: ${{ matrix.arch }}/rust
env:
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
ARROW_TEST_DATA: /__w/arrow-rs/arrow-rs/testing/data
PARQUET_TEST_DATA: /__w/arrow-rs/arrow-rs/parquet-testing/data
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- name: Run tests
run: |
# run tests on all workspace members with default feature list
cargo test
- name: Re-run tests with all supported features
run: |
cargo test -p arrow --features=force_validate,prettyprint
- name: Run examples
run: |
# Test arrow examples
cargo run --example builders
cargo run --example dynamic_types
cargo run --example read_csv
cargo run --example read_csv_infer_schema
- name: Test compilation of arrow library crate with different feature combinations
run: |
cargo check -p arrow
cargo check -p arrow --no-default-features
- name: Test compilation of arrow targets with different feature combinations
run: |
cargo check -p arrow --all-targets
cargo check -p arrow --no-default-features --all-targets
cargo check -p arrow --no-default-features --all-targets --features test_utils
- name: Re-run tests on arrow-flight with all features
run: |
cargo test -p arrow-flight --all-features
- name: Re-run tests on parquet crate with all features
run: |
cargo test -p parquet --all-features
- name: Test compilation of parquet library crate with different feature combinations
run: |
cargo check -p parquet
cargo check -p parquet --no-default-features
cargo check -p parquet --no-default-features --features arrow
cargo check -p parquet --all-features
- name: Test compilation of parquet targets with different feature combinations
run: |
cargo check -p parquet --all-targets
cargo check -p parquet --no-default-features --all-targets
cargo check -p parquet --no-default-features --features arrow --all-targets
- name: Test compilation of parquet_derive macro with different feature combinations
run: |
cargo check -p parquet_derive
# test the --features "simd" of the arrow crate. This requires nightly.
linux-test-simd:
name: Test SIMD on AMD64 Rust ${{ matrix.rust }}
runs-on: ubuntu-latest
strategy:
matrix:
arch: [ amd64 ]
rust: [ nightly ]
container:
image: ${{ matrix.arch }}/rust
env:
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
ARROW_TEST_DATA: /__w/arrow-rs/arrow-rs/testing/data
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- name: Run tests
run: |
cargo test -p arrow --features "simd"
- name: Check compilation with simd features
run: |
cargo check -p arrow --features simd
cargo check -p arrow --features simd --all-targets
windows-and-macos:
name: Test on ${{ matrix.os }} Rust ${{ matrix.rust }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ windows-latest, macos-latest ]
rust: [ stable ]
steps:
- uses: actions/checkout@v2
with:
submodules: true
# TODO: this won't cache anything, which is expensive. Setup this action
# with a OS-dependent path.
- name: Setup Rust toolchain
run: |
rustup toolchain install ${{ matrix.rust }}
rustup default ${{ matrix.rust }}
- name: Run tests
shell: bash
run: |
export ARROW_TEST_DATA=$(pwd)/testing/data
export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
# do not produce debug symbols to keep memory usage down
export RUSTFLAGS="-C debuginfo=0"
cargo test
clippy:
name: Clippy
needs: [ linux-build-lib ]
runs-on: ubuntu-latest
strategy:
matrix:
arch: [ amd64 ]
rust: [ stable ]
container:
image: ${{ matrix.arch }}/rust
env:
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- name: Setup Clippy
run: |
rustup component add clippy
- name: Run clippy
run: |
cargo clippy --features test_common --features prettyprint --features=async --all-targets --workspace -- -D warnings
check_benches:
name: Check Benchmarks (but don't run them)
runs-on: ubuntu-latest
strategy:
matrix:
arch: [ amd64 ]
rust: [ stable ]
container:
image: ${{ matrix.arch }}/rust
env:
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- name: Check benchmarks
run: |
cargo check --benches --workspace --features test_common,prettyprint,async,experimental
lint:
name: Lint (cargo fmt)
runs-on: ubuntu-latest
container:
image: amd64/rust
steps:
- uses: actions/checkout@v2
- name: Setup toolchain
run: |
rustup toolchain install stable
rustup default stable
rustup component add rustfmt
- name: Run
run: cargo fmt --all -- --check
coverage:
name: Coverage
runs-on: ubuntu-latest
strategy:
matrix:
arch: [ amd64 ]
rust: [ stable ]
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup Rust toolchain
run: |
rustup toolchain install ${{ matrix.rust }}
rustup default ${{ matrix.rust }}
- name: Cache Cargo
uses: actions/cache@v3
with:
path: /home/runner/.cargo
# this key is not equal because the user is different than on a container (runner vs github)
key: cargo-coverage-cache3-
- name: Cache Rust dependencies
uses: actions/cache@v3
with:
path: /home/runner/target
# this key is not equal because coverage uses different compilation flags.
key: ${{ runner.os }}-${{ matrix.arch }}-target-coverage-cache3-${{ matrix.rust }}-
- name: Run coverage
run: |
export CARGO_HOME="/home/runner/.cargo"
export CARGO_TARGET_DIR="/home/runner/target"
export ARROW_TEST_DATA=$(pwd)/testing/data
export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
rustup toolchain install stable
rustup default stable
cargo install --version 0.18.2 cargo-tarpaulin
cargo tarpaulin --all --out Xml
- name: Report coverage
continue-on-error: true
run: bash <(curl -s https://codecov.io/bash)
# test the arrow crate builds against wasm32 in stable rust
wasm32-build:
name: Build wasm32 on AMD64 Rust ${{ matrix.rust }}
runs-on: ubuntu-latest
strategy:
matrix:
arch: [ amd64 ]
rust: [ nightly ]
container:
image: ${{ matrix.arch }}/rust
env:
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
ARROW_TEST_DATA: /__w/arrow-rs/arrow-rs/testing/data
PARQUET_TEST_DATA: /__w/arrow/arrow/parquet-testing/data
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Cache Cargo
uses: actions/cache@v3
with:
path: /github/home/.cargo
key: cargo-wasm32-cache3-
- name: Cache Rust dependencies
uses: actions/cache@v3
with:
path: /github/home/target
key: ${{ runner.os }}-${{ matrix.arch }}-target-wasm32-cache3-${{ matrix.rust }}
- name: Setup Rust toolchain for WASM
run: |
rustup toolchain install ${{ matrix.rust }}
rustup override set ${{ matrix.rust }}
rustup target add wasm32-unknown-unknown
rustup target add wasm32-wasi
- name: Build arrow crate
run: |
cd arrow
cargo build --no-default-features --features=csv,ipc,simd --target wasm32-unknown-unknown
cargo build --no-default-features --features=csv,ipc,simd --target wasm32-wasi
# test doc links still work
docs:
name: Docs are clean on AMD64 Rust ${{ matrix.rust }}
runs-on: ubuntu-latest
strategy:
matrix:
arch: [ amd64 ]
rust: [ nightly ]
container:
image: ${{ matrix.arch }}/rust
env:
# Disable full debug symbol generation to speed up CI build and keep memory down
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
RUSTDOCFLAGS: "-Dwarnings"
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Install python dev
run: |
apt update
apt install -y libpython3.9-dev
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
with:
rust-version: ${{ matrix.rust }}
- name: Run cargo doc
run: |
cargo doc --document-private-items --no-deps --workspace --all-features