From 9650be0aefe20b923927153b62ea4a62ff9853e6 Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Fri, 13 Nov 2020 15:53:04 -0800 Subject: [PATCH 01/12] chore: automate running benchmarks Uses Github actions to run benchmarks. --- .github/workflows/bench.yml | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/bench.yml diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 00000000000..c6b80f480a1 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,48 @@ +name: Benchmark +on: + push: + branches: + - master + +jobs: + benchmark: + name: Performance regression check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Rust + run: rustup update stable + + # Run benchmark with `go test -bench` and stores the output to a file + - name: Run benchmark + run: cargo bench | tee ../output.txt + working-directory: benches + + # Download previous benchmark result from cache (if exists) + - name: Download previous benchmark data + uses: actions/cache@v1 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + + # Run `github-action-benchmark` action + - name: Store benchmark result + uses: rhysd/github-action-benchmark@v1 + with: + name: Tokio Benchmarks + # What benchmark tool the output.txt came from + tool: 'cargo' + # Where the output from the benchmark tool is stored + output-file-path: output.txt + # Where the previous data file is stored + external-data-json-path: ./cache/benchmark-data.json + # Workflow will fail when an alert happens + fail-on-alert: true + # GitHub API token to make a commit comment + github-token: ${{ secrets.GITHUB_TOKEN }} + # Enable alert commit comment + comment-on-alert: true + alert-comment-cc-users: '@tokio-rs/maintainers' + auto-push: true + + # Upload the updated cache file for the next job by actions/cache \ No newline at end of file From c438a4c3a60a18fb747c00506b80f9b77331ee71 Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Fri, 13 Nov 2020 15:55:48 -0800 Subject: [PATCH 02/12] try running from the branch --- .github/workflows/bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index c6b80f480a1..6a0c242ed23 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -2,7 +2,7 @@ name: Benchmark on: push: branches: - - master + - bench-ci jobs: benchmark: From 043b41e8d1e7c2c7d1840f954cf4857d5a1729dd Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Fri, 13 Nov 2020 15:58:04 -0800 Subject: [PATCH 03/12] ignore CI for now --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bf58b4a9af..5aea16afd0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,8 @@ on: push: branches: ["master"] + branches-ignore: + - 'bench-ci' pull_request: branches: ["master"] From 39f7d89f11d4b89fabff4a84646f2a46a4eb1ea0 Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Fri, 13 Nov 2020 16:01:17 -0800 Subject: [PATCH 04/12] try again --- .github/workflows/bench.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 6a0c242ed23..af0f963a659 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -34,8 +34,8 @@ jobs: tool: 'cargo' # Where the output from the benchmark tool is stored output-file-path: output.txt - # Where the previous data file is stored - external-data-json-path: ./cache/benchmark-data.json + # # Where the previous data file is stored + # external-data-json-path: ./cache/benchmark-data.json # Workflow will fail when an alert happens fail-on-alert: true # GitHub API token to make a commit comment From 3a758470548e2dffffa3ce0f8e8904ac0a5ddc59 Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Fri, 13 Nov 2020 16:13:41 -0800 Subject: [PATCH 05/12] try matrix build --- .github/workflows/bench.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index af0f963a659..92476c3b1ec 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -8,6 +8,11 @@ jobs: benchmark: name: Performance regression check runs-on: ubuntu-latest + strategy: + matrix: + bench: + - mpsc + - scheduler steps: - uses: actions/checkout@v2 - name: Install Rust @@ -15,7 +20,7 @@ jobs: # Run benchmark with `go test -bench` and stores the output to a file - name: Run benchmark - run: cargo bench | tee ../output.txt + run: cargo bench --bench ${{ matrix.bench }} | tee ../output.txt working-directory: benches # Download previous benchmark result from cache (if exists) From 118b44a3fd58015855ba3033c55e2d33bdda9306 Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Fri, 13 Nov 2020 16:17:01 -0800 Subject: [PATCH 06/12] tweak --- .github/workflows/bench.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 92476c3b1ec..43fa6c4d27e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -6,7 +6,7 @@ on: jobs: benchmark: - name: Performance regression check + name: Benchmark runs-on: ubuntu-latest strategy: matrix: @@ -34,7 +34,7 @@ jobs: - name: Store benchmark result uses: rhysd/github-action-benchmark@v1 with: - name: Tokio Benchmarks + name: ${{ matrix.bench }} # What benchmark tool the output.txt came from tool: 'cargo' # Where the output from the benchmark tool is stored From aee637485f38a052e5ba204d0327ab6d452d2946 Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Fri, 13 Nov 2020 16:23:48 -0800 Subject: [PATCH 07/12] tweaks --- .github/workflows/bench.yml | 6 ++++-- benches/{scheduler.rs => rt_multi_threaded.rs} | 0 benches/{mpsc.rs => sync_mpsc.rs} | 0 3 files changed, 4 insertions(+), 2 deletions(-) rename benches/{scheduler.rs => rt_multi_threaded.rs} (100%) rename benches/{mpsc.rs => sync_mpsc.rs} (100%) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 43fa6c4d27e..a20186d087a 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -11,8 +11,10 @@ jobs: strategy: matrix: bench: - - mpsc - - scheduler + - rt_multi_thread + - sync_mpsc + - sync_rwlock + - sync_semaphore steps: - uses: actions/checkout@v2 - name: Install Rust diff --git a/benches/scheduler.rs b/benches/rt_multi_threaded.rs similarity index 100% rename from benches/scheduler.rs rename to benches/rt_multi_threaded.rs diff --git a/benches/mpsc.rs b/benches/sync_mpsc.rs similarity index 100% rename from benches/mpsc.rs rename to benches/sync_mpsc.rs From 26bffa6fc4b7c833eb1ad5888f530d449168044e Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Fri, 13 Nov 2020 16:31:34 -0800 Subject: [PATCH 08/12] bump ci From 83565495515fe252af2bde975986bfd75df65263 Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Fri, 13 Nov 2020 16:35:19 -0800 Subject: [PATCH 09/12] no push --- .github/workflows/bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index a20186d087a..c97e444bbb5 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -50,6 +50,6 @@ jobs: # Enable alert commit comment comment-on-alert: true alert-comment-cc-users: '@tokio-rs/maintainers' - auto-push: true + # auto-push: true # Upload the updated cache file for the next job by actions/cache \ No newline at end of file From 1c6a96f0321ad4aecf9fea496a6ddf83bbf3a55e Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Fri, 13 Nov 2020 16:43:07 -0800 Subject: [PATCH 10/12] try again --- .github/workflows/bench.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index c97e444bbb5..1851d47a90e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: bench: - - rt_multi_thread + - rt_multi_threaded - sync_mpsc - sync_rwlock - sync_semaphore @@ -50,6 +50,6 @@ jobs: # Enable alert commit comment comment-on-alert: true alert-comment-cc-users: '@tokio-rs/maintainers' - # auto-push: true + auto-push: true # Upload the updated cache file for the next job by actions/cache \ No newline at end of file From af032dbf59195f5a637c14fd8805f45cce8c8563 Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Fri, 13 Nov 2020 16:47:49 -0800 Subject: [PATCH 11/12] try again --- benches/Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benches/Cargo.toml b/benches/Cargo.toml index cca0ece5eb8..d0383f36dce 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -17,13 +17,13 @@ path = "spawn.rs" harness = false [[bench]] -name = "mpsc" -path = "mpsc.rs" +name = "sync_mpsc" +path = "sync_mpsc.rs" harness = false [[bench]] -name = "scheduler" -path = "scheduler.rs" +name = "rt_multi_threaded" +path = "rt_multi_threaded.rs" harness = false From 9bc89aaa9e259aebaa683abcc8087891a9f6ece7 Mon Sep 17 00:00:00 2001 From: Carl Lerche Date: Fri, 13 Nov 2020 16:59:54 -0800 Subject: [PATCH 12/12] prepare for PR --- .github/workflows/bench.yml | 2 +- .github/workflows/ci.yml | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 1851d47a90e..98925986d13 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -2,7 +2,7 @@ name: Benchmark on: push: branches: - - bench-ci + - master jobs: benchmark: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5aea16afd0d..9bf58b4a9af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,6 @@ on: push: branches: ["master"] - branches-ignore: - - 'bench-ci' pull_request: branches: ["master"]