Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Issue #14599: Create performance regression test CI
- Loading branch information
Showing
3 changed files
with
169 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
#!/bin/bash | ||
|
||
set -e | ||
|
||
# max difference tolerance in % | ||
THRESHOLD_PERCENTAGE=10 | ||
# baseline of execution time in seconds | ||
BASELINE_SECONDS=415.72 | ||
|
||
# JDK version | ||
JDK_VERSION=17 | ||
# sample project path | ||
SAMPLE_PROJECT="./.ci-temp/jdk$JDK_VERSION" | ||
# suppression file | ||
SUPPRESSION_FILE="./config/projects-to-test/openjdk$JDK_VERSION-excluded.files" | ||
# config file | ||
CONFIG_FILE="./config/benchmark-config.xml" | ||
|
||
# execute a command and time it | ||
# $TEST_COMMAND: command being timed | ||
time_command() { | ||
# execute the command and time it | ||
/usr/bin/time -o time.temp -q -f "%e" "$@" &>result.tmp | ||
|
||
cat time.temp | ||
} | ||
|
||
# execute the benchmark a few times to calculate the average metrics | ||
# $JAR_PATH: path of the jar file being benchmarked | ||
execute_benchmark() { | ||
local JAR_PATH=$1 | ||
if [ -z "$JAR_PATH" ]; then | ||
echo "Missing JAR_PATH as an argument." | ||
exit 1 | ||
fi | ||
|
||
local TOTAL_SECONDS=0 | ||
local NUM_EXECUTIONS=3 | ||
|
||
[ ! -d "$SAMPLE_PROJECT" ] && | ||
echo "Directory $SAMPLE_PROJECT DOES NOT exist." | exit 1 | ||
|
||
# add suppressions to config file | ||
sed -i '/ <!-- Filters -->/r' $SUPPRESSION_FILE \ | ||
./config/benchmark-config.xml | ||
|
||
for ((i = 0; i < NUM_EXECUTIONS; i++)); do | ||
local CMD=(java -jar "$JAR_PATH" -c "$CONFIG_FILE" \ | ||
-x .git -x module-info.java "$SAMPLE_PROJECT") | ||
local BENCHMARK=($(time_command "${CMD[@]}")) | ||
TOTAL_SECONDS=$(echo "$TOTAL_SECONDS + $BENCHMARK" | bc) | ||
done | ||
|
||
# average execution time in patch | ||
local AVERAGE_IN_SECONDS=$(echo "scale=2; $TOTAL_SECONDS / $NUM_EXECUTIONS" | bc) | ||
echo "$AVERAGE_IN_SECONDS" | ||
} | ||
|
||
# compare baseline and patch benchmarks | ||
# $EXECUTION_TIME_SECONDS execution time of the patch | ||
compare_results() { | ||
local EXECUTION_TIME_SECONDS=$1 | ||
if [ -z "$EXECUTION_TIME_SECONDS" ]; then | ||
echo "Missing EXECUTION_TIME_SECONDS as an argument." | ||
exit 1 | ||
fi | ||
# Calculate percentage difference for execution time | ||
local DEVIATION_IN_SECONDS=$(echo "scale=4; \ | ||
((${EXECUTION_TIME_SECONDS} - ${BASELINE_SECONDS}) / ${BASELINE_SECONDS}) * 100" | bc) | ||
echo "Execution Time Difference: $DEVIATION_IN_SECONDS%" | ||
|
||
# Check if differences exceed the maximum allowed difference | ||
if (( $(echo "$DEVIATION_IN_SECONDS > $THRESHOLD_PERCENTAGE" | bc -l) )); then | ||
echo "Difference exceeds the maximum allowed difference (${DEVIATION_IN_SECONDS}% \ | ||
> ${THRESHOLD_PERCENTAGE}%)!" | ||
exit 1 | ||
else | ||
echo "Difference is within the maximum allowed difference (${DEVIATION_IN_SECONDS}% \ | ||
<= ${THRESHOLD_PERCENTAGE}%)." | ||
exit 0 | ||
fi | ||
} | ||
|
||
# package patch | ||
mvn -e --no-transfer-progress -Passembly,no-validations package | ||
|
||
# start benchmark | ||
echo "Benchmark launching..." | ||
AVERAGE_IN_SECONDS="$(execute_benchmark "$(find "./target/" -type f -name "checkstyle-*-all.jar")")" | ||
|
||
# print the command execution result | ||
echo "================ MOST RECENT COMMAND RESULT =================" | ||
cat result.tmp | ||
|
||
echo "===================== BENCHMARK SUMMARY ====================" | ||
echo "Execution Time Baseline: ${BASELINE_SECONDS} s" | ||
echo "Average Execution Time: ${AVERAGE_IN_SECONDS} s" | ||
echo "============================================================" | ||
|
||
# compare result with baseline | ||
compare_results "$AVERAGE_IN_SECONDS" | ||
|
||
exit $? |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
##################################################################################### | ||
# GitHub Action to test performance regression. | ||
# | ||
# Workflow starts when: | ||
# 1) push to master | ||
# 2) PR created or pushed | ||
# | ||
##################################################################################### | ||
name: Check-Performance-Regression | ||
|
||
on: | ||
push: | ||
branches: | ||
- master | ||
pull_request: | ||
branches: '*' | ||
|
||
concurrency: | ||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | ||
cancel-in-progress: true | ||
|
||
jobs: | ||
test: | ||
if: github.repository == 'checkstyle/checkstyle' | ||
runs-on: ubuntu-latest | ||
steps: | ||
- name: Set up JDK 11 | ||
uses: actions/setup-java@v4 | ||
with: | ||
java-version: 11 | ||
distribution: 'temurin' | ||
|
||
- name: Checkout patch | ||
uses: actions/checkout@v4 | ||
with: | ||
ref: ${{ github.event.pull_request.head.sha }} | ||
path: ./checkstyle | ||
|
||
- name: Clone JDK 17 Repo | ||
uses: actions/checkout@v4 | ||
with: | ||
repository: openjdk/jdk17 | ||
path: ./checkstyle/.ci-temp/jdk17 | ||
|
||
- name: Setup local maven cache | ||
uses: actions/cache@v4 | ||
with: | ||
path: ~/.m2/repository | ||
key: checkstyle-maven-cache-${{ hashFiles('**/pom.xml') }} | ||
|
||
- name: Run performance test | ||
run: | | ||
cd checkstyle | ||
bash ./.ci/check-performance-regression.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
<?xml version="1.0"?> | ||
<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN" | ||
"http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd"> | ||
<module name="Checker"> | ||
<!-- Filters --> | ||
|
||
<property name="severity" value="ignore"/> | ||
<module name="TreeWalker"> | ||
<module name="NoCodeInFileCheck"> | ||
</module> | ||
</module> | ||
</module> |