Issue #14599: Create performance regression test CI

.ci/benchmark-config.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0"?>
+<!DOCTYPE module PUBLIC "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+  "http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd">
+<module name="Checker">
+  <!-- Filters -->
+  <property name="severity" value="ignore"/>
+  <module name="TreeWalker">
+    <module name="NoCodeInFileCheck">
+    </module>
+  </module>
+</module>

.ci/check-performance-regression.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+set -e
+
+# max difference tolerance in %
+MAX_DIFFERENCE=10
+# baseline of execution time in second
+EXECUTION_TIME_BASELINE=665
+
+# sample project path
+SAMPLE_PROJECT="./.ci/jdk17"
+CONFIG_FILE="./.ci/benchmark-config.xml"
+
+# run a command and time it
+# $TEST_COMMAND: command being timed
+time_command() {
+  # Run the command with time
+  local TEST_COMMAND=$1
+  if [ -z "$TEST_COMMAND" ]; then
+        echo "Missing TEST_COMMAND as an argument."
+        exit 1
+    fi
+  # shellcheck disable=SC2086
+  local TIME_OUTPUT=$(command time -p $TEST_COMMAND 2>&1)
+  # Extract execution time
+  local EXECUTION_TIME=$(echo "$TIME_OUTPUT" | awk '/real/ {print $2}')
+
+  echo "${EXECUTION_TIME}"
+}
+
+# insert all the filters into the config file
+generate_config() {
+  sed -i '/  <!-- Filters -->/r ./config/projects-to-test/openjdk17-excluded.files' \
+        $CONFIG_FILE
+}
+
+# run the benchmark a few times to calculate the average metrics
+# $JAR_PATH: path of the jar file being benchmarked
+run_benchmark() {
+  local JAR_PATH=$1
+  if [ -z "$JAR_PATH" ]; then
+      echo "Missing JAR_PATH as an argument."
+      exit 1
+  fi
+
+  local TOTAL_TIME=0
+  local NUM_RUNS=3
+
+  [ ! -d "$SAMPLE_PROJECT" ] &&
+    echo "Directory $SAMPLE_PROJECT DOES NOT exists." | exit 1
+
+  for ((i = 1; i <= NUM_RUNS; i++)); do
+    local CMD="java -jar $JAR_PATH -c $CONFIG_FILE -x .git -x module-info.java $SAMPLE_PROJECT"
+    local BENCHMARK=($(time_command "$CMD"))
+    TOTAL_TIME=$(echo "$TOTAL_TIME + ${BENCHMARK}" | bc)
+  done
+
+  # average execution time in patch
+  local AVG_TIME=$(echo "scale=2; $TOTAL_TIME /   $NUM_RUNS" | bc)
+  echo "$AVG_TIME"
+}
+
+# compare baseline and patch benchmarks
+# $EXECUTION_TIME execution time of the patch
+compare_results() {
+  local EXECUTION_TIME=$1
+  if [ -z "$EXECUTION_TIME" ]; then
+        echo "Missing EXECUTION_TIME as an argument."
+        exit 1
+    fi
+  # Calculate percentage difference for execution time
+  local EXECUTION_TIME_DIFFERENCE=$(echo "scale=4; \
+    ((${EXECUTION_TIME} - ${EXECUTION_TIME_BASELINE}) / ${EXECUTION_TIME_BASELINE}) * 100" | bc)
+  echo "Execution Time Difference: $EXECUTION_TIME_DIFFERENCE%"
+
+  # Check if differences exceed the maximum allowed difference
+  if (( $(echo "$EXECUTION_TIME_DIFFERENCE > $MAX_DIFFERENCE" | bc -l) )); then
+    echo "Difference exceeds the maximum allowed difference (${EXECUTION_TIME_DIFFERENCE}% \
+     > ${MAX_DIFFERENCE}%)!"
+    exit 1
+  else
+    echo "Difference is within the maximum allowed difference (${EXECUTION_TIME_DIFFERENCE}% \
+     <= ${MAX_DIFFERENCE}%)."
+    exit 0
+  fi
+}
+
+# package patch
+export MAVEN_OPTS='-Xmx2000m'
+mvn -e --no-transfer-progress -Passembly,no-validations package
+
+generate_config
+
+# run benchmark
+echo "Benchmark launching..."
+AVG_TIME="$(run_benchmark "$(find "./target/" -type f -name "checkstyle-*-all.jar")")"
+echo "===================== BENCHMARK SUMMARY ===================="
+echo "Average Execution Time: ${AVG_TIME} s"
+echo "============================================================"
+
+# compare result with baseline
+compare_results "$AVG_TIME"
+exit $?

.github/workflows/check-performance-regression.yml

@@ -0,0 +1,53 @@
+#####################################################################################
+# GitHub Action to test performance regression.
+#
+# Workflow starts when:
+# 1) push to master
+# 2) PR created or pushed
+#
+#####################################################################################
+name: Check-Performance-Regression
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches: '*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: 'temurin'
+
+      - name: Checkout patch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: ./checkstyle
+
+      - name: Clone JDK 17 Repo
+        uses: actions/checkout@v4
+        with:
+          repository: openjdk/jdk17
+          path: ./checkstyle/.ci/jdk17
+
+      - name: Setup local maven cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: checkstyle-maven-cache-${{ hashFiles('**/pom.xml') }}
+
+      - name: Run performance test
+        run: |
+          cd checkstyle
+          bash ./.ci/check-performance-regression.sh

config/checkstyle-non-main-files-checks.xml

@@ -19,7 +19,7 @@
   <module name="RegexpOnFilename">
     <property name="id" value="executablesLocation"/>
     <property name="folderPattern" value="[\\/].ci([\\/]|$)"/>
-    <property name="fileNamePattern" value="\.(bat|cmd|groovy|sh|md)$"/>
+    <property name="fileNamePattern" value="\.(bat|cmd|groovy|sh|md|xml)$"/>
     <property name="match" value="false"/>
     <message key="regexp.filename.mismatch"
       value="CI folder should only contain executables files"/>