Fix/bench results (#800)

* fix(bench): Include clone3 in thread count

It's a newer syscall interface and gets counted seperately in strace results.

* fix(bench): Update totals line parsing

The column offsets didn't match up when usecs/call is included in the totals.

* fix(bench): Enforce locale, avoiding parse errors

On my local machine, it used an NL locale changing the strace numbers to use commas.
Adding this env var avoids that problem.

* fix(bench): Use matrix for the rust toolchain version

* fix(bench): Avoid multiple runs

* fix(bench): Ensure backtraces

* fix(bench): Simplify caching similar to tauri

* fix(bench): Linter change

.github/workflows/bench.yml

@@ -6,23 +6,32 @@ on:
       - dev
   workflow_dispatch:
 
+env:
+  RUST_BACKTRACE: 1
+  CARGO_PROFILE_DEV_DEBUG: 0 # This would add unnecessary bloat to the target folder, decreasing cache efficiency.
+  LC_ALL: en_US.UTF-8 # This prevents strace from changing it's number format to use commas.
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   bench:
     strategy:
       fail-fast: false
       matrix:
-        rust_version: [stable]
+        rust: [nightly]
         platform:
           - { target: x86_64-unknown-linux-gnu, os: ubuntu-latest }
 
     runs-on: ${{ matrix.platform.os }}
 
     steps:
       - uses: actions/checkout@v2
-      - name: install stable
+      - name: install ${{ matrix.rust }}
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: nightly
+          toolchain: ${{ matrix.rust }}
           override: true
           components: rust-src
           target: ${{ matrix.platform.target }}
@@ -42,52 +51,11 @@ jobs:
           sudo dpkg -i hyperfine_1.11.0_amd64.deb
           pip install memory_profiler
 
-      - name: get current date
-        run: echo "CURRENT_DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
-
-      - name: cache cargo registry
-        uses: actions/cache@v2.1.4
-        with:
-          path: ~/.cargo/registry
-          # Add date to the cache to keep it up to date
-          key: ${{ matrix.platform }}-stable-cargo-registry-${{ hashFiles('Cargo.toml') }}-${{ env.CURRENT_DATE }}
-          # Restore from outdated cache for speed
-          restore-keys: |
-            ${{ matrix.platform }}-stable-cargo-registry-${{ hashFiles('Cargo.toml') }}
-            ${{ matrix.platform }}-stable-cargo-registry-
-
-      - name: cache cargo index
-        uses: actions/cache@v2.1.4
-        with:
-          path: ~/.cargo/git
-          # Add date to the cache to keep it up to date
-          key: ${{ matrix.platform }}-stable-cargo-index-${{ hashFiles('Cargo.toml') }}-${{ env.CURRENT_DATE }}
-          # Restore from outdated cache for speed
-          restore-keys: |
-            ${{ matrix.platform }}-stable-cargo-index-${{ hashFiles('Cargo.toml') }}
-            ${{ matrix.platform }}-stable-cargo-index-
-
-      - name: cache cargo target
-        uses: actions/cache@v2
-        with:
-          path: target
-          # Add date to the cache to keep it up to date
-          key: ${{ matrix.platform }}-stable-cargo-core-${{ hashFiles('Cargo.toml') }}-${{ env.CURRENT_DATE }}
-          # Restore from outdated cache for speed
-          restore-keys: |
-            ${{ matrix.platform }}-stable-cargo-core-${{ hashFiles('Cargo.toml') }}
-            ${{ matrix.platform }}-stable-cargo-core-
-
-      - name: cache cargo `bench/tests` target
-        uses: actions/cache@v2
+      - uses: Swatinem/rust-cache@v2
         with:
-          path: bench/tests/target
-          # Add date to the cache to keep it up to date
-          key: ubuntu-latest-nightly-cargo-benches-${{ hashFiles('bench/tests/Cargo.toml') }}-${{ env.CURRENT_DATE }}
-          # Restore from outdated cache for speed
-          restore-keys: |
-            ${{ matrix.platform }}-stable-cargo-benches-${{ hashFiles('bench/tests/Cargo.toml') }}
-            ${{ matrix.platform }}-stable-cargo-benches-
+          workspaces: |
+            .
+            bench/tests
 
       - name: run benchmarks
         run: |

bench/src/run_benchmark.rs

@@ -64,7 +64,9 @@ fn run_strace_benchmarks(new_data: &mut utils::BenchResult) -> Result<()> {
     file.as_file_mut().read_to_string(&mut output)?;
 
     let strace_result = utils::parse_strace_output(&output);
-    let clone = strace_result.get("clone").map(|d| d.calls).unwrap_or(0) + 1;
+    let clone = 1
+      + strace_result.get("clone").map(|d| d.calls).unwrap_or(0)
+      + strace_result.get("clone3").map(|d| d.calls).unwrap_or(0);
     let total = strace_result.get("total").unwrap().calls;
     thread_count.insert(name.to_string(), clone);
     syscall_count.insert(name.to_string(), total);

bench/src/utils.rs

@@ -157,16 +157,31 @@ pub fn parse_strace_output(output: &str) -> HashMap<String, StraceOutput> {
   }
 
   let total_fields = total_line.split_whitespace().collect::<Vec<_>>();
-  summary.insert(
-    "total".to_string(),
-    StraceOutput {
-      percent_time: str::parse::<f64>(total_fields[0]).unwrap(),
-      seconds: str::parse::<f64>(total_fields[1]).unwrap(),
-      usecs_per_call: None,
-      calls: str::parse::<u64>(total_fields[2]).unwrap(),
-      errors: str::parse::<u64>(total_fields[3]).unwrap(),
-    },
-  );
+
+  match total_fields.len() {
+    // Old format, has no usecs/call
+    5 => summary.insert(
+      "total".to_string(),
+      StraceOutput {
+        percent_time: str::parse::<f64>(total_fields[0]).unwrap(),
+        seconds: str::parse::<f64>(total_fields[1]).unwrap(),
+        usecs_per_call: None,
+        calls: str::parse::<u64>(total_fields[2]).unwrap(),
+        errors: str::parse::<u64>(total_fields[3]).unwrap(),
+      },
+    ),
+    6 => summary.insert(
+      "total".to_string(),
+      StraceOutput {
+        percent_time: str::parse::<f64>(total_fields[0]).unwrap(),
+        seconds: str::parse::<f64>(total_fields[1]).unwrap(),
+        usecs_per_call: Some(str::parse::<u64>(total_fields[2]).unwrap()),
+        calls: str::parse::<u64>(total_fields[3]).unwrap(),
+        errors: str::parse::<u64>(total_fields[4]).unwrap(),
+      },
+    ),
+    _ => panic!("Unexpected total field count: {}", total_fields.len()),
+  };
 
   summary
 }