diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8837d8fc31..da20ed1d41 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -11,8 +11,6 @@ jobs: Prepare: runs-on: ubuntu-latest outputs: - crates: ${{ steps.get_matrix.outputs.crates }} - deps: ${{ steps.get_matrix.outputs.deps }} nightly_version: ${{ steps.read_toolchain.outputs.nightly_version }} steps: - name: Checkout Crate @@ -20,220 +18,261 @@ jobs: - name: Read nightly version id: read_toolchain run: echo "nightly_version=$(cat nightly-version)" >> $GITHUB_OUTPUT - - name: Prepare tests - id: get_matrix - run: contrib/get_matrix.sh - Stable: - needs: Prepare + Stable: # 2 jobs, one per manifest. name: Test - stable toolchain runs-on: ubuntu-latest strategy: fail-fast: false matrix: - crate: ${{ fromJson(needs.Prepare.outputs.crates) }} - dep: ${{ fromJson(needs.Prepare.outputs.deps) }} - task: [test, docs, feature_matrix, dup_deps] + dep: [minimal, recent] steps: - - name: Checkout Crate + - name: "Checkout repo" uses: actions/checkout@v4 - - name: Checkout Toolchain - # https://github.com/dtolnay/rust-toolchain + - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: Set dependencies + - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - - name: Running test script - run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }} + - name: "Run test script" + run: ./contrib/run_task.sh stable - Beta: + Nightly: # 2 jobs, one per manifest. + name: Test - nightly toolchain needs: Prepare - name: Test - beta toolchain runs-on: ubuntu-latest strategy: fail-fast: false matrix: - crate: ${{ fromJson(needs.Prepare.outputs.crates) }} - dep: ${{ fromJson(needs.Prepare.outputs.deps) }} - task: [test] + dep: [minimal, recent] steps: - - name: Checkout Crate + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: ${{ needs.Prepare.outputs.nightly_version }} + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./contrib/run_task.sh nightly + + MSRV: # 2 jobs, one per manifest. + name: Test - 1.56.1 toolchain + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [minimal, recent] + steps: + - name: "Checkout repo" uses: actions/checkout@v4 - - name: Checkout Toolchain - uses: dtolnay/rust-toolchain@beta - - name: Set dependencies + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.56.1" + - name: "Copy lock file" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - - name: Running test script - run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }} + - name: "Run test script" + run: ./contrib/run_task.sh msrv - Nightly: + Lint: + name: Lint - nightly toolchain needs: Prepare - name: Test - nightly toolchain runs-on: ubuntu-latest strategy: fail-fast: false matrix: - crate: ${{ fromJson(needs.Prepare.outputs.crates) }} - dep: ${{ fromJson(needs.Prepare.outputs.deps) }} - task: [test, lint, bench, docsrs] + dep: [recent] steps: - - name: Checkout Crate + - name: "Checkout repo" uses: actions/checkout@v4 - - name: Checkout Toolchain + - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: toolchain: ${{ needs.Prepare.outputs.nightly_version }} - name: Install clippy run: rustup component add clippy - - name: Set dependencies + - name: "Set dependencies" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - - name: Running test script - run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }} + - name: "Run test script" + run: ./contrib/run_task.sh lint - MSRV: - needs: Prepare - name: Test - 1.56.1 toolchain + Docs: + name: Docs - stable toolchain runs-on: ubuntu-latest strategy: fail-fast: false matrix: - crate: ${{ fromJson(needs.Prepare.outputs.crates) }} - dep: ${{ fromJson(needs.Prepare.outputs.deps) }} - task: [test, feature_matrix] + dep: [recent] steps: - - name: Checkout Crate + - name: "Checkout repo" uses: actions/checkout@v4 - - name: Checkout Toolchain + - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable + - name: "Copy lock file" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./contrib/run_task.sh docs + + Docsrs: + name: Docs - nightly toolchain + needs: Prepare + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [recent] + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@v1 with: - toolchain: "1.56.1" - - name: Set dependencies + toolchain: ${{ needs.Prepare.outputs.nightly_version }} + - name: "Set dependencies" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./contrib/run_task.sh docsrs + + Bench: + name: Bench - nightly toolchain + needs: Prepare + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + dep: [recent] + steps: + - name: "Checkout repo" + uses: actions/checkout@v4 + - name: "Select toolchain" + uses: dtolnay/rust-toolchain@v1 + with: + toolchain: ${{ needs.Prepare.outputs.nightly_version }} + - name: "Copy lock file" run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock - - name: Running test script - run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }} + - name: "Run test script" + run: ./contrib/run_task.sh bench Arch32bit: name: Test 32-bit version runs-on: ubuntu-latest steps: - - name: Checkout Crate + - name: "Checkout repo" uses: actions/checkout@v4 - - name: Checkout Toolchain + - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: Add architecture i386 + - name: "Add architecture i386" run: sudo dpkg --add-architecture i386 - - name: Install i686 gcc + - name: "Install i686 gcc" run: sudo apt-get update -y && sudo apt-get install -y gcc-multilib - - name: Install target + - name: "Install target" run: rustup target add i686-unknown-linux-gnu - - name: Run test on i686 + - name: "Run test on i686" run: cargo test --target i686-unknown-linux-gnu Cross: - name: Cross test + name: Cross test - stable toolchain if: ${{ !github.event.act }} runs-on: ubuntu-latest steps: - - name: Checkout Crate + - name: "Checkout repo" uses: actions/checkout@v4 - - name: Checkout Toolchain + - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: Install target + - name: "Install target" run: rustup target add s390x-unknown-linux-gnu - - name: install cross + - name: "Install cross" run: cargo install cross --locked - - name: run cross test + - name: "Run cross test" run: cross test --target s390x-unknown-linux-gnu Embedded: + name: Embedded - nightly toolchain needs: Prepare runs-on: ubuntu-latest env: RUSTFLAGS: "-C link-arg=-Tlink.x" CARGO_TARGET_THUMBV7M_NONE_EABI_RUNNER: "qemu-system-arm -cpu cortex-m3 -machine mps2-an385 -nographic -semihosting-config enable=on,target=native -kernel" steps: - - name: Checkout + - name: "Checkout repo" uses: actions/checkout@v4 - - name: Set up QEMU + - name: "Set up QEMU" run: sudo apt update && sudo apt install -y qemu-system-arm gcc-arm-none-eabi - - name: Checkout Toolchain + - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: toolchain: ${{ needs.Prepare.outputs.nightly_version }} targets: thumbv7m-none-eabi - - name: Install src + - name: "Install rust-src" run: rustup component add rust-src - - name: Run bitcoin/embedded + - name: "Run bitcoin/embedded" run: cd bitcoin/embedded && cargo run --target thumbv7m-none-eabi - - name: Run hashes/embedded no alloc + - name: "Run hashes/embedded no alloc" run: cd hashes/embedded && cargo run --target thumbv7m-none-eabi - - name: Run hashes/embedded with alloc + - name: "Run hashes/embedded with alloc" run: cd hashes/embedded && cargo run --target thumbv7m-none-eabi --features=alloc - ASAN: + ASAN: # hashes crate only. + name: ASAN - nightly toolchain needs: Prepare - name: Address sanitizer # hashes crate only. runs-on: ubuntu-latest strategy: fail-fast: false matrix: - crate: [hashes] - dep: ${{ fromJson(needs.Prepare.outputs.deps) }} - task: [asan] + dep: [recent] steps: - - name: Checkout Crate + - name: "Checkout repo" uses: actions/checkout@v4 - - name: Checkout Toolchain + - name: "Select toolchain" uses: dtolnay/rust-toolchain@v1 with: toolchain: ${{ needs.Prepare.outputs.nightly_version }} - name: Install src run: rustup component add rust-src - - name: Running address sanitizer - run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }} + - name: "Copy lock file" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./contrib/run_task.sh asan - WASM: - needs: Prepare - name: WebAssembly Build # hashes crate only. + WASM: # hashes crate only. + name: WASM - stable toolchain runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - crate: [hashes] - dep: ${{ fromJson(needs.Prepare.outputs.deps) }} - task: [wasm] + # Note we do not use the recent lock file for wasm testing. steps: - - name: Checkout Crate + - name: "Checkout repo" uses: actions/checkout@v4 - - name: Checkout Toolchain + - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: Running WASM build - run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }} + - name: "Run test script" + run: ./contrib/run_task.sh wasm - Schemars: - needs: Prepare - name: Schemars + Schemars: # hashes crate only. + name: Schemars - stable toolchain runs-on: ubuntu-latest strategy: fail-fast: false matrix: - crate: [hashes] - dep: ${{ fromJson(needs.Prepare.outputs.deps) }} - task: [schemars] + dep: [recent] steps: - - name: Checkout Crate + - name: "Checkout repo" uses: actions/checkout@v4 - - name: Checkout Toolchain + - name: "Select toolchain" uses: dtolnay/rust-toolchain@stable - - name: Running schemars test - run: ./contrib/run_task.sh ${{ matrix.crate }} ${{ matrix.task }} + - name: "Copy lock file" + run: cp Cargo-${{ matrix.dep }}.lock Cargo.lock + - name: "Run test script" + run: ./contrib/run_task.sh schemars Kani: + name: Kani codegen - stable toolchain runs-on: ubuntu-20.04 steps: - - name: 'Checkout your code.' + - name: "Checkout repo" uses: actions/checkout@v4 - - - name: 'Kani build proofs' + - name: "Build Kani proofs" uses: model-checking/kani-github-action@v1.1 with: - args: '--only-codegen' + args: "--only-codegen" diff --git a/base58/contrib/test_vars.sh b/base58/contrib/test_vars.sh index 7701af2071..0c25edc7b9 100644 --- a/base58/contrib/test_vars.sh +++ b/base58/contrib/test_vars.sh @@ -6,5 +6,5 @@ FEATURES_WITH_STD="" # Test all these features without "std" enabled. FEATURES_WITHOUT_STD="" -# Run and lint these examples. +# Run these examples. EXAMPLES="" diff --git a/bitcoin/contrib/test_vars.sh b/bitcoin/contrib/test_vars.sh index 26ef2480cf..e1d0231467 100644 --- a/bitcoin/contrib/test_vars.sh +++ b/bitcoin/contrib/test_vars.sh @@ -6,5 +6,5 @@ FEATURES_WITH_STD="rand-std serde secp-recovery bitcoinconsensus-std base64 orde # Test all these features without "std" or "alloc" enabled. FEATURES_WITHOUT_STD="rand serde secp-recovery bitcoinconsensus base64 ordered" -# Run and lint these examples. +# Run these examples. EXAMPLES="ecdsa-psbt:std,bitcoinconsensus sign-tx-segwit-v0:rand-std sign-tx-taproot:rand-std taproot-psbt:bitcoinconsensus,rand-std sighash:std" diff --git a/contrib/run_task.sh b/contrib/run_task.sh index 094289bea4..424dd902a8 100755 --- a/contrib/run_task.sh +++ b/contrib/run_task.sh @@ -1,13 +1,52 @@ -#!/bin/env bash +#!/usr/bin/env bash +# +# We are attempting to run max 20 parallel jobs using GitHub actions (usage limit for free tier). +# +# ref: https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration +# +# The minimal/recent manifests are handled by CI (rust.yaml). +# +# Jobs (shell functions) that get run twice, once for each manifest: +# +# 1+2 stable +# 3+4 nightly +# 5+6 msrv +# +# Jobs (shell functions) that get run once: +# +# 7 lint +# 8 docs +# 9 docsrs +# 10 bench +# 11 asan +# 12 wasm +# 13 schemars +# +# Jobs run directly by rust.yml workflow: +# +# 0 Prepare +# +# 14 Arch32bit +# 15 Cross +# 16 Embedded +# 17 Kani +# +# Jobs run directly from other workflows: +# +# 18 Coveralls - run by coveralls.yml +# 19 release - run by release.yml +# 20 labeler - run by manage-pr.yml set -euox pipefail +REPO_DIR=$(git rev-parse --show-toplevel) +CRATES=("bitcoin" "hashes" "internals" "io" "units" "base58") + # Make all cargo invocations verbose. export CARGO_TERM_VERBOSE=true main() { - crate="$1" - task="$2" + local task="$1" check_required_commands @@ -17,34 +56,27 @@ main() { locale env - cd "$crate" - - # Building the fuzz crate is more-or-less just a sanity check. - if [ "$crate" == "fuzz" ] - then - cargo --locked build - exit 0 - fi - - # Every crate must define EXAMPLES. - . contrib/test_vars.sh || exit 1 - case $task in - test) - do_test - ;; - - feature_matrix) - do_feature_matrix - ;; - - lint) - do_lint - ;; - - dup_deps) - do_dup_deps - ;; + # 2 jobs each for these (one for each lock file). + stable) + # Test, run examples, do feature matrix. + # crate/contrib/test_vars.sh is sourced in this function. + build_and_test + ;; + + nightly) + build_and_test + ;; + + msrv) + build_and_test + ;; + + # 1 job each for these. + lint) + do_lint + do_dup_deps + ;; docs) build_docs_with_stable_toolchain @@ -54,27 +86,51 @@ main() { build_docs_with_nightly_toolchain ;; + bench) + do_bench + ;; + wasm) + # hashes crate only. do_wasm ;; asan) + # hashes crate only - hashes/contrib/test_vars.sh is sourced in this function. do_asan ;; - bench) - do_bench - ;; - schemars) + # hashes crate only. do_schemars ;; - *) - err "Error: unknown task $task" - ;; + + *) + err "Error: unknown task $task" + ;; esac } +# Build and test for each crate, done with each toolchain. +build_and_test() { + # Building the fuzz crate is more-or-less just a sanity check. + pushd "$REPO_DIR/fuzz" > /dev/null + cargo --locked build + popd > /dev/null + + for crate in "${CRATES[@]}"; do + pushd "$REPO_DIR/$crate" > /dev/null + + # Set crate specific variables. + . contrib/test_vars.sh || exit 1 + + do_test + do_feature_matrix + + popd > /dev/null + done +} + do_test() { # Use the current (recent/minimal) lock file. local cargo="cargo --locked" @@ -83,15 +139,15 @@ do_test() { $cargo build $cargo test - for example in $EXAMPLES; do - name="$(echo "$example" | cut -d ':' -f 1)" - features="$(echo "$example" | cut -d ':' -f 2)" - $cargo run --example "$name" --features="$features" + for example in $EXAMPLES; do # EXAMPLES is set in contrib/test_vars.sh + name="$(echo "$example" | cut -d ':' -f 1)" + features="$(echo "$example" | cut -d ':' -f 2)" + $cargo run --example "$name" --features="$features" done if [ -e ./contrib/extra_tests.sh ]; then - ./contrib/extra_tests.sh + ./contrib/extra_tests.sh fi } @@ -103,7 +159,8 @@ do_feature_matrix() { $cargo build --no-default-features $cargo test --no-default-features - # All crates have a "std" feature. + # All crates have a "std" feature and FEATURES_WITH_STD is set in + # contrib/test_vars.sh loop_features "std" "$FEATURES_WITH_STD" # All but `bitcoin` crate have an "alloc" feature, this tests it @@ -146,20 +203,15 @@ loop_features() { fi } -# Lint the workspace then the individual crate examples. +# Lint the workspace. do_lint() { need_nightly - - # Use the current (recent/minimal) lock file. local cargo="cargo --locked" - $cargo clippy --workspace -- -D warnings - - for example in $EXAMPLES; do - name=$(echo "$example" | cut -d ':' -f 1) - features=$(echo "$example" | cut -d ':' -f 2) - $cargo clippy --example "$name" --features="$features" -- -D warnings - done + # Lint various feature combinations to try and catch mistakes in feature gating. + $cargo clippy --workspace --all-targets --keep-going -- -D warnings + $cargo clippy --workspace --all-targets --all-features --keep-going -- -D warnings + $cargo clippy --workspace --all-targets --no-default-features --keep-going -- -D warnings } # We should not have any duplicate dependencies. This catches mistakes made upgrading dependencies @@ -173,7 +225,7 @@ do_dup_deps() { # Only show the actual duplicated deps, not their reverse tree, then # whitelist the 'syn' crate which is duplicated but it's not our fault. # - # Temporarily allow 2 versions of `hashes` and `hex` while we upgrade. + # Temporarily allow 2 versions of `hashes`, `internals`, and `hex` while we upgrade. cargo tree --target=all --all-features --duplicates \ | grep '^[0-9A-Za-z]' \ | grep -v 'syn' \ @@ -193,7 +245,6 @@ do_dup_deps() { build_docs_with_nightly_toolchain() { need_nightly local cargo="cargo --locked" - RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" $cargo doc --all-features } @@ -201,20 +252,45 @@ build_docs_with_nightly_toolchain() { # above this checks that we feature guarded docs imports correctly. build_docs_with_stable_toolchain() { local cargo="cargo +stable --locked" - RUSTDOCFLAGS="-D warnings" $cargo doc --all-features } +# Bench only works with a non-stable toolchain (nightly, beta). +do_bench() { + for crate in bitcoin hashes; do + pushd "$REPO_DIR/$crate" > /dev/null + RUSTFLAGS='--cfg=bench' cargo bench + popd > /dev/null + done +} + +# This is only relevant for hashes. +do_schemars() { + pushd "$REPO_DIR/hashes/extended_tests/schemars" > /dev/null + cargo test + popd > /dev/null +} + +# Note we do not use the recent lock file or `--locked` when running the wasm tests. do_wasm() { + pushd "$REPO_DIR/hashes" > /dev/null + clang --version && CARGO_TARGET_DIR=wasm cargo install --force wasm-pack && printf '\n[target.wasm32-unknown-unknown.dev-dependencies]\nwasm-bindgen-test = "0.3"\n' >> Cargo.toml && printf '\n[lib]\ncrate-type = ["cdylib", "rlib"]\n' >> Cargo.toml && CC=clang-9 wasm-pack build && CC=clang-9 wasm-pack test --node; + + popd > /dev/null } do_asan() { + pushd "$REPO_DIR/hashes" > /dev/null + + # Set ASAN_FEATURES + . contrib/test_vars.sh || exit 1 + cargo clean CC='clang -fsanitize=address -fno-omit-frame-pointer' \ RUSTFLAGS='-Zsanitizer=address -Clinker=clang -Cforce-frame-pointers=yes' \ @@ -226,17 +302,8 @@ do_asan() { # CC='clang -fsanitize=memory -fno-omit-frame-pointer' \ # RUSTFLAGS='-Zsanitizer=memory -Zsanitizer-memory-track-origins -Cforce-frame-pointers=yes' \ # cargo test --lib --no-default-features --features="$ASAN_FEATURES" -Zbuild-std --target x86_64-unknown-linux-gnu -} -# Bench only works with a non-stable toolchain (nightly, beta). -do_bench() { - RUSTFLAGS='--cfg=bench' cargo bench -} - -# This is only relevant for hashes. -do_schemars() { - cd "extended_tests/schemars" > /dev/null - cargo test + popd > /dev/null } # Check all the commands we use are present in the current environment. diff --git a/hashes/contrib/test_vars.sh b/hashes/contrib/test_vars.sh index a75854cd33..5e89c21e77 100644 --- a/hashes/contrib/test_vars.sh +++ b/hashes/contrib/test_vars.sh @@ -9,5 +9,5 @@ FEATURES_WITHOUT_STD="alloc serde small-hash" # Run address sanitizer with these features. ASAN_FEATURES="std io serde" -# Run and lint these examples. +# Run these examples. EXAMPLES="" diff --git a/hashes/src/sha256.rs b/hashes/src/sha256.rs index 2d03fdd70e..8affa07b18 100644 --- a/hashes/src/sha256.rs +++ b/hashes/src/sha256.rs @@ -1027,10 +1027,9 @@ mod tests { #[cfg(target_arch = "wasm32")] mod wasm_tests { - extern crate wasm_bindgen_test; - use self::wasm_bindgen_test::*; use super::*; - #[wasm_bindgen_test] + #[test] + #[wasm_bindgen_test::wasm_bindgen_test] fn sha256_tests() { test(); midstate(); diff --git a/internals/contrib/test_vars.sh b/internals/contrib/test_vars.sh index 22b5b7e32e..759f5afbd9 100644 --- a/internals/contrib/test_vars.sh +++ b/internals/contrib/test_vars.sh @@ -6,5 +6,5 @@ FEATURES_WITH_STD="serde" # Test all these features without "std" enabled. FEATURES_WITHOUT_STD="alloc serde" -# Run and lint these examples. +# Run these examples. EXAMPLES="" diff --git a/io/contrib/test_vars.sh b/io/contrib/test_vars.sh index 9eac7dcc21..0317fc3b6d 100644 --- a/io/contrib/test_vars.sh +++ b/io/contrib/test_vars.sh @@ -6,5 +6,5 @@ FEATURES_WITH_STD="" # Test all these features without "std" enabled. FEATURES_WITHOUT_STD="alloc" -# Run and lint these examples. +# Run these examples. EXAMPLES="" diff --git a/units/contrib/test_vars.sh b/units/contrib/test_vars.sh index 22b5b7e32e..759f5afbd9 100644 --- a/units/contrib/test_vars.sh +++ b/units/contrib/test_vars.sh @@ -6,5 +6,5 @@ FEATURES_WITH_STD="serde" # Test all these features without "std" enabled. FEATURES_WITHOUT_STD="alloc serde" -# Run and lint these examples. +# Run these examples. EXAMPLES=""