Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Issue #14599: Create performance regression test CI
- Loading branch information
Showing
5 changed files
with
186 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
Running benchmark 1/3... | ||
================== BENCHMARK RESULT #1 ================== | ||
Execution Time: 8.82 s | ||
============================================================ | ||
Running benchmark 2/3... | ||
================== BENCHMARK RESULT #2 ================== | ||
Execution Time: 8.37 s | ||
============================================================ | ||
Running benchmark 3/3... | ||
================== BENCHMARK RESULT #3 ================== | ||
Execution Time: 8.50 s | ||
============================================================ | ||
===================== BENCHMARK SUMMARY ==================== | ||
Average Execution Time: 8.56 s | ||
============================================================ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
#!/bin/bash | ||
|
||
set -e | ||
|
||
# max % difference tolerance | ||
MAX_DIFFERENCE=10 | ||
|
||
# parse benchmark result at the given path | ||
parse_benchmark_result() { | ||
BENCHMARK_PATH=$1 | ||
EXECUTION_TIME=$(awk '/Average Execution Time:/ {print $4}' "$BENCHMARK_PATH") | ||
|
||
local RESULT_ARRAY=($EXECUTION_TIME) | ||
echo "${RESULT_ARRAY[@]}" | ||
} | ||
|
||
# compare baseline and patch benchmarks | ||
compare_results() { | ||
# Calculate percentage difference for execution time | ||
EXECUTION_TIME_DIFFERENCE=$(echo "scale=4; \ | ||
((${PATCH[0]} - ${BASELINE[0]}) / ${BASELINE[0]}) * 100" | bc) | ||
echo "Execution Time Difference: $EXECUTION_TIME_DIFFERENCE%" | ||
|
||
# Check if differences exceed the maximum allowed difference | ||
if (( $(echo "$EXECUTION_TIME_DIFFERENCE > $MAX_DIFFERENCE" | bc -l) )); then | ||
echo "Differences exceed the maximum allowed difference (${MAX_DIFFERENCE}%)!" | ||
exit 1 | ||
else | ||
echo "Differences are within the maximum allowed difference." | ||
exit 0 | ||
fi | ||
} | ||
|
||
# parse baseline benchmark | ||
BASELINE=($(parse_benchmark_result "./.ci/baseline.benchmark")) | ||
|
||
# package patch | ||
export MAVEN_OPTS='-Xmx2000m' | ||
mvn -e --no-transfer-progress -Passembly,no-validations package | ||
|
||
# run benchmark and parse result | ||
JAR_FILE=$(find "./target/" -type f -name "checkstyle-*-all.jar") | ||
bash ./.ci/run-benchmark.sh "${JAR_FILE}" | ||
PATCH=($(parse_benchmark_result "./patch.benchmark")) | ||
|
||
# compare two metrics | ||
compare_results | ||
exit $? |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
#!/bin/bash | ||
|
||
set -e | ||
|
||
# check if jar path is provided | ||
JAR_PATH=$1 | ||
if [ -z "$JAR_PATH" ]; then | ||
echo "Usage: $0 <target jar path>" | ||
exit 1 | ||
fi | ||
|
||
# Sample project path | ||
SAMPLE_PROJECT="./.ci/jdk22" | ||
|
||
# generate benchmark config file | ||
generate_config() { | ||
local CONFIG="<?xml version=\"1.0\"?> | ||
<!DOCTYPE module PUBLIC \"-//Puppy Crawl//DTD Check | ||
Configuration 1.3//EN\" | ||
\"http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd\"> | ||
<module name=\"Checker\"> | ||
<module name=\"TreeWalker\"/> | ||
</module>" | ||
echo "$CONFIG" > ./.ci/benchmark-config.xml | ||
} | ||
|
||
# run a command and time it | ||
time_command() { | ||
# Run the command with time | ||
TEST_COMMAND=$1 | ||
# shellcheck disable=SC2086 | ||
TIME_OUTPUT=$(command time -p $TEST_COMMAND 2>&1) | ||
# Extract execution time | ||
EXECUTION_TIME=$(echo "$TIME_OUTPUT" | awk '/real/ {print $2}') | ||
|
||
local RESULT_ARRAY=($EXECUTION_TIME) | ||
echo "${RESULT_ARRAY[@]}" | ||
} | ||
|
||
# run the benchmark a few times to calculate the average metrics | ||
run_benchmark() { | ||
local TOTAL_TIME=0 | ||
local NUM_RUNS=3 | ||
|
||
[ ! -d "$SAMPLE_PROJECT" ] && | ||
echo "Directory $SAMPLE_PROJECT DOES NOT exists." | exit 1 | ||
|
||
generate_config | ||
for ((i = 1; i <= NUM_RUNS; i++)); do | ||
echo "Running benchmark ${i}/${NUM_RUNS}..." | ||
local CMD="java -jar $JAR_PATH -c ./.ci/benchmark-config.xml -x .git $SAMPLE_PROJECT" | ||
local BENCHMARK=($(time_command "$CMD")) | ||
TOTAL_TIME=$(echo "$TOTAL_TIME + ${BENCHMARK[0]}" | bc) | ||
echo "================== BENCHMARK RESULT #${i} ==================" | ||
echo "Execution Time: ${BENCHMARK[0]} s" | ||
echo "============================================================" | ||
done | ||
|
||
local AVG_TIME=$(echo "scale=2; $TOTAL_TIME / $NUM_RUNS" | bc) | ||
|
||
echo "===================== BENCHMARK SUMMARY ====================" | ||
echo "Average Execution Time: ${AVG_TIME} s" | ||
echo "============================================================" | ||
} | ||
|
||
# save the benchmark result | ||
run_benchmark | tee ./patch.benchmark | ||
exit $? | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
##################################################################################### | ||
# GitHub Action to test performance regression. | ||
# | ||
# Workflow starts when: | ||
# 1) push to master | ||
# 2) PR created or pushed | ||
# | ||
##################################################################################### | ||
name: Check-Performance-Regression | ||
|
||
on: | ||
push: | ||
branches: | ||
- master | ||
pull_request: | ||
branches: '*' | ||
|
||
concurrency: | ||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | ||
cancel-in-progress: true | ||
|
||
jobs: | ||
test: | ||
runs-on: ubuntu-latest | ||
steps: | ||
- name: Set up JDK 11 | ||
uses: actions/setup-java@v4 | ||
with: | ||
java-version: 11 | ||
distribution: 'temurin' | ||
|
||
- name: Checkout patch | ||
uses: actions/checkout@v4 | ||
with: | ||
ref: ${{ github.event.pull_request.head.sha }} | ||
path: ./checkstyle | ||
|
||
- name: Clone JDK 22 Repo | ||
uses: actions/checkout@v4 | ||
with: | ||
repository: openjdk/jdk22 | ||
path: ./checkstyle/.ci/jdk22 | ||
|
||
- name: Setup local maven cache | ||
uses: actions/cache@v4 | ||
with: | ||
path: ~/.m2/repository | ||
key: checkstyle-maven-cache-${{ hashFiles('**/pom.xml') }} | ||
|
||
- name: Run performance test | ||
run: | | ||
cd checkstyle | ||
bash ./.ci/check-performance-regression.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters