Issue #10272: Upgrade Java Grammar from ANTLR2 to ANTLR4

checkstyle · Jul 19, 2021 · 2f5c283 · 2f5c283
1 parent b557d01
commit 2f5c283
Show file tree

Hide file tree

Showing 130 changed files with 30,102 additions and 3,501 deletions.
diff --git a/.ci/jsoref-spellchecker/whitelist.words b/.ci/jsoref-spellchecker/whitelist.words
@@ -143,6 +143,7 @@ breadcrumbs
 bsr
 Bsubscribe
 BTest
+btnfr
 buf
 bugdatabase
 bugfixes
@@ -276,10 +277,12 @@ dalvik
 Daniil
 daniilyar
 Darguments
+darray
 Datasource
 DATETIME
 daveho
 davidwalsh
+DBFF
 Dcheckstyle
 dcm
 DDDD
@@ -305,6 +308,7 @@ detailnodetreestringprinter
 devops
 Dexec
 dfa
+DFFF
 Dfile
 dfn
 Dfoo
@@ -715,13 +719,14 @@ kbd
 kclee
 KDoc
 keygen
+Kochurkin
 konstantinos
 Kordas
 Kotlin
 lambdabodylength
 lambdaparametername
 lbc
-LBRACK
+lbrack
 lbt
 lcurly
 Leanback
@@ -1047,7 +1052,7 @@ qwe
 qwerty
 rawtypes
 rb
-RBRACK
+rbrack
 rcurly
 rdiachenko
 RDz

diff --git a/.ci/no-exception-test.sh b/.ci/no-exception-test.sh
@@ -60,7 +60,7 @@ openjdk14-with-checks-nonjavadoc-error)
   cd .ci-temp/contribution/checkstyle-tester
   cp ../../../.ci/openjdk-projects-to-test-on.config openjdk-projects-to-test-on.config
   sed -i '/  <!-- Filters -->/r ../../../.ci/openjdk14-excluded.files' checks-nonjavadoc-error.xml
-  export MAVEN_OPTS="-Xmx2048m"
+  export MAVEN_OPTS="-Xmx4096m"
   groovy ./diff.groovy --listOfProjects openjdk-projects-to-test-on.config \
       --mode single --allowExcludes \
       --patchConfig checks-nonjavadoc-error.xml \
@@ -83,7 +83,7 @@ no-exception-lucene-and-others-javadoc)
   sed -i'' 's/#protonpack/protonpack/' projects-for-no-exception-javadoc.config
   sed -i'' 's/#jOOL/jOOL/' projects-for-no-exception-javadoc.config
   sed -i'' 's/#lucene-solr/lucene-solr/' projects-for-no-exception-javadoc.config
-  export MAVEN_OPTS="-Xmx2048m"
+  export MAVEN_OPTS="-Xmx3072m"
   groovy ./diff.groovy --listOfProjects projects-for-no-exception-javadoc.config \
       --patchConfig checks-only-javadoc-error.xml \
       --mode single --allowExcludes -xm "-Dcheckstyle.failsOnError=false \

diff --git a/.ci/pitest.sh b/.ci/pitest.sh
@@ -172,6 +172,23 @@ pitest-utils)
   checkPitestReport "${ignoredItems[@]}"
   ;;
 
+pitest-JavaAstVisitor)
+  mvn -e -P$1 clean test org.pitest:pitest-maven:mutationCoverage;
+  declare -a ignoredItems=(
+  "JavaAstVisitor.java.html:<td class='covered'><pre><span  class='survived'>        copy.setColumnNo(ast.getColumnNo());</span></pre></td></tr>"
+  "JavaAstVisitor.java.html:<td class='covered'><pre><span  class='survived'>        copy.setLineNo(ast.getLineNo());</span></pre></td></tr>"
+  "JavaAstVisitor.java.html:<td class='covered'><pre><span  class='survived'>        detailAst.initialize(token);</span></pre></td></tr>"
+  "JavaAstVisitor.java.html:<td class='covered'><pre><span  class='survived'>        DetailAstPair.addASTChild(currentAST, visit(ctx.annotations()));</span></pre></td></tr>"
+  "JavaAstVisitor.java.html:<td class='covered'><pre><span  class='survived'>        else if (ctx.anno.isEmpty()) {</span></pre></td></tr>"
+  "JavaAstVisitor.java.html:<td class='covered'><pre><span  class='survived'>        enumDef.setLineNo(enumDef.getFirstChild().getLineNo());</span></pre></td></tr>"
+  "JavaAstVisitor.java.html:<td class='covered'><pre><span  class='survived'>        if (binOpList.isEmpty()) {</span></pre></td></tr>"
+  "JavaAstVisitor.java.html:<td class='covered'><pre><span  class='survived'>        type.setLineNo(type.getFirstChild().getLineNo());</span></pre></td></tr>"
+  "JavaAstVisitor.java.html:<td class='covered'><pre><span  class='survived'>        while (firstExpression instanceof CheckstyleJavaParser.BinOpContext) {</span></pre></td></tr>"
+  );
+  checkPitestReport "${ignoredItems[@]}"
+  ;;
+
+
 # pitesttyle-gui)
 #   mvn -e -P$1 clean test org.pitest:pitest-maven:mutationCoverage;
 #   # post validation is skipped, we do not test gui throughly

diff --git a/.github/workflows/no-exception-workflow.yml b/.github/workflows/no-exception-workflow.yml
@@ -99,3 +99,22 @@ jobs:
       run: mvn -e --no-transfer-progress clean install -Pno-validations
 
     - run: ./.ci/no-exception-test.sh no-exception-only-javadoc
+
+  no-exception-openjdk14:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up JDK 8
+      uses: actions/setup-java@v1
+      with:
+        java-version: 8
+
+    - name: Install dependencies
+      run: sudo apt install groovy
+
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Install checkstyle
+      run: mvn -e --no-transfer-progress clean install -Pno-validations
+
+    - run: .ci/no-exception-test.sh openjdk14-with-checks-nonjavadoc-error
diff --git a/.github/workflows/pitest.yml b/.github/workflows/pitest.yml
@@ -0,0 +1,21 @@
+name: pitest workflow
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  pitest9:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Set up JDK 8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 8
+
+      - run: ./.ci/pitest.sh pitest-JavaAstVisitor
diff --git a/.gitignore b/.gitignore
@@ -44,3 +44,7 @@ replay_pid*
 .ci/jsoref-spellchecker/spelling-unknown-word-splitter.pl
 .ci/jsoref-spellchecker/unknown.words
 .ci/jsoref-spellchecker/english.words
+
+# antlr4 grammar generates these
+**/gen
+**/JavaLexer.tokens
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
@@ -78,8 +78,6 @@ blocks:
                 # until https://github.com/checkstyle/checkstyle/issues/9248
                 # - mvn -e clean install -Pno-validations
                 #    && .ci/validation.sh no-violation-test-configurate
-                - mvn -e --no-transfer-progress clean install -Pno-validations
-                    && .ci/no-exception-test.sh openjdk14-with-checks-nonjavadoc-error
           commands:
             - sem-version java 11
             - echo "eval of CMD is starting";

diff --git a/config/ant-phase-verify.xml b/config/ant-phase-verify.xml
@@ -116,12 +116,8 @@
           <!-- Do not validate possible source code remnants after regression testing -->
           <exclude name=".ci-temp/**/*"/>
 
-          <!-- Exclude because printed AST has line length > 100 characters -->
-          <exclude name="**/InputJava14InstanceofWithPatternMatchingAST.txt"/>
-          <exclude name="**/InputJava14Records.txt"/>
-          <exclude name="**/InputJava14TextBlocks.txt"/>
-          <exclude name="**/InputJava14SwitchExpression.txt"/>
-          <exclude name="**/InputPatternVariableWithModifiers.txt"/>
+          <!-- Exclude because printed AST is verified by matching checkstyle output -->
+          <exclude name="**/grammar/**/*.txt"/>
 
         </fileset>
       </path>

diff --git a/config/checkstyle_checks.xml b/config/checkstyle_checks.xml
@@ -378,8 +378,6 @@
                 value="org.xml.sax.SAXException, org.xml.sax.SAXParseException,
                        org.apache.commons.beanutils.ConversionException,
                        org.antlr.v4.runtime.misc.ParseCancellationException,
-                       antlr.RecognitionException, antlr.TokenStreamException,
-                       antlr.TokenStreamRecognitionException, antlr.ANTLRException,
                        java.lang.StringBuffer"/>
     </module>
     <module name="IllegalThrows"/>

diff --git a/config/checkstyle_non_main_files_suppressions.xml b/config/checkstyle_non_main_files_suppressions.xml
@@ -17,6 +17,8 @@
              files="[\\/]test[\\/].*[\\/]asttreestringprinter[\\/]InputAstTreeStringPrinterFullOfSinglelineComments\.java"/>
   <suppress checks="NewlineAtEndOfFile"
              files="src[\\/]test[\\/]resources[\\/].*[\\/]InputNoCodeInFile6\.java"/>
+  <suppress checks="NewlineAtEndOfFile"
+             files="src[\\/]test[\\/]resources[\\/].*[\\/]InputAntlr4AstRegressionCassandraInputWithComments\.java"/>
 
   <!-- Grammar specific input file, should have exact structure to reproduce the case. -->
   <suppress checks="NewlineAtEndOfFile"

diff --git a/config/import-control.xml b/config/import-control.xml
@@ -46,8 +46,8 @@
   <allow class="com.puppycrawl.tools.checkstyle.XpathFileGeneratorAuditListener"
          local-only="true"/>
 
-  <file name="DetailAstImpl|JavaParser" regex="true">
-    <allow pkg="antlr"/>
+  <file name="DetailAstImpl|JavaParser|JavaAstVisitor|CheckstyleParserErrorStrategy" regex="true">
+    <allow pkg="org.antlr.v4.runtime"/>
   </file>
   <file name="JavadocDetailNodeParser">
     <allow pkg="org.antlr.v4.runtime"/>
@@ -56,8 +56,8 @@
     <allow class="java.math.BigInteger"/>
   </file>
   <file name="ParserUtil">
-    <allow class="antlr.Token"/>
     <allow class="com.puppycrawl.tools.checkstyle.DetailAstImpl"/>
+    <allow class="org.antlr.v4.runtime.CommonToken"/>
   </file>
 
   <file name="DefaultLogger">
@@ -111,6 +111,12 @@
     <allow class="com.puppycrawl.tools.checkstyle.Checker" local-only="true"/>
     <!-- allowed till https://github.com/checkstyle/checkstyle/issues/3817 -->
     <allow pkg="com.puppycrawl.tools.checkstyle.utils"/>
+    <file name="DetailAST">
+      <allow class="org.antlr.v4.runtime.Token"/>
+    </file>
+    <file name="TokenTypes">
+      <allow class="org.antlr.v4.runtime.Recognizer"/>
+    </file>
   </subpackage>
 
   <subpackage name="checks">
@@ -166,7 +172,6 @@
 
   <subpackage name="gui" strategyOnMismatch="disallowed">
     <!-- until https://github.com/checkstyle/checkstyle/issues/3095 -->
-    <allow pkg="antlr"/>
     <allow class="com.puppycrawl.tools.checkstyle.DetailAstImpl"/>
     <allow class="com.puppycrawl.tools.checkstyle.xpath.XpathQueryGenerator" />
 

diff --git a/config/pmd.xml b/config/pmd.xml
@@ -320,7 +320,8 @@
            JavadocMethodCheck is in the process of being rewritten. -->
       <property name="violationSuppressXPath"
                 value="//ClassOrInterfaceDeclaration[@SimpleName='PackageObjectFactory']
-          | //ClassOrInterfaceDeclaration[@SimpleName='JavadocMethodCheck']"/>
+          | //ClassOrInterfaceDeclaration[@SimpleName='JavadocMethodCheck']
+          | //ClassOrInterfaceDeclaration[@SimpleName='JavaAstVisitor']"/>
     </properties>
   </rule>
   <rule ref="category/java/design.xml/ExcessiveClassLength">
@@ -331,7 +332,7 @@
       <property name="violationSuppressXPath"
                 value="//ClassOrInterfaceDeclaration[@SimpleName='JavadocTokenTypes'
         or @SimpleName='TokenTypes' or @SimpleName='RequireThisCheck'
-        or @SimpleName='JavadocMethodCheck']"/>
+        or @SimpleName='JavadocMethodCheck' or @SimpleName='JavaAstVisitor']"/>
     </properties>
   </rule>
   <rule ref="category/java/design.xml/ExcessiveParameterList">
@@ -386,4 +387,11 @@
     <!-- Not configurable, decreases readability. -->
     <exclude name="UseStringBufferForStringAppends"/>
   </rule>
+
+  <rule ref="category/java/design.xml/ExcessivePublicCount">
+    <properties>
+      <property name="violationSuppressXPath"
+                value="//ClassOrInterfaceDeclaration[@SimpleName='JavaAstVisitor']"/>
+    </properties>
+  </rule>
 </ruleset>
diff --git a/config/suppressions.xml b/config/suppressions.xml
@@ -645,4 +645,15 @@
   <suppress id="ImportControlTest"
             files="[\\/]src[\\/]test[\\/]java[\\/]com[\\/]puppycrawl[\\/]tools[\\/]checkstyle[\\/]xpath[\\/]XpathQueryGeneratorTest.java"/>
 
+  <!-- required by JavaAstVisitor design -->
+  <suppress checks="MethodCount"
+            files="[\\/]src[\\/]main[\\/]java[\\/]com[\\/]puppycrawl[\\/]tools[\\/]checkstyle[\\/]JavaAstVisitor.java"/>
+  <suppress checks="FileLength"
+            files="[\\/]src[\\/]main[\\/]java[\\/]com[\\/]puppycrawl[\\/]tools[\\/]checkstyle[\\/]JavaAstVisitor.java"/>
+  <suppress checks="ClassFanOutComplexity"
+            files="[\\/]src[\\/]main[\\/]java[\\/]com[\\/]puppycrawl[\\/]tools[\\/]checkstyle[\\/]JavaAstVisitor.java"/>
+
+  <!-- DetailAST API requires large number of methods -->
+  <suppress checks="MethodCount"
+            files="[\\/]src[\\/]main[\\/]java[\\/]com[\\/]puppycrawl[\\/]tools[\\/]checkstyle[\\/]DetailAstImpl.java"/>
 </suppressions>