Use signed ints for ATN serialization not uint16, except for java

.circleci/scripts/run-tests-cpp.sh

@@ -3,5 +3,6 @@
 set -euo pipefail
 
 pushd runtime-testsuite
+export MAVEN_OPTS="-Xmx8g"
 mvn -Dparallel=classes -DthreadCount=4 -Dtest=cpp.** test
 popd

.circleci/scripts/run-tests-dart.sh

@@ -6,5 +6,6 @@ dart --version
 
 pushd runtime-testsuite
   echo "running maven tests..."
+  export MAVEN_OPTS="-Xmx8g"
   mvn -Dparallel=classes -DthreadCount=4 -Dtest=dart.** test
 popd

.circleci/scripts/run-tests-dotnet.sh

@@ -3,5 +3,6 @@
 set -euo pipefail
 
 pushd runtime-testsuite
+export MAVEN_OPTS="-Xmx8g"
 mvn -Dparallel=classes -DthreadCount=4 -Dtest=csharp.** test
 popd

.circleci/scripts/run-tests-go.sh

@@ -6,5 +6,6 @@ go version
 
 pushd runtime-testsuite
     echo "running maven tests..."
+    export MAVEN_OPTS="-Xmx8g"
     mvn -Dparallel=classes -DthreadCount=4 -Dtest=go.** test
 popd

.circleci/scripts/run-tests-javascript.sh

@@ -15,6 +15,7 @@ popd
 pushd runtime-testsuite
 
   echo "running maven tests..."
+  export MAVEN_OPTS="-Xmx8g"
   mvn -Dtest=javascript.** test
   RESULT+=$?
 

.circleci/scripts/run-tests-php.sh

@@ -7,5 +7,6 @@ php -v
 php_path=$(which php)
 pushd runtime-testsuite
   echo "running maven tests..."
+  export MAVEN_OPTS="-Xmx8g"
   mvn -DPHP_PATH="${php_path}" -Dparallel=classes -DthreadCount=4 -Dtest=php.** test
 popd

.circleci/scripts/run-tests-python2.sh

@@ -17,5 +17,6 @@ python2 --version
 
 pushd runtime-testsuite
   echo "running maven tests..."
+  export MAVEN_OPTS="-Xmx8g"
   mvn -Dparallel=classes -DthreadCount=4 -Dtest=python2.** test
 popd

.circleci/scripts/run-tests-python3.sh

@@ -17,5 +17,6 @@ python3 --version
 
 pushd runtime-testsuite
   echo "running maven tests..."
+  export MAVEN_OPTS="-Xmx8g"
   mvn -Dparallel=classes -DthreadCount=4 -Dtest=python3.** test
 popd

.circleci/scripts/run-tests-swift.sh

@@ -17,5 +17,6 @@ set -euo pipefail
 
 pushd runtime-testsuite
   echo "running maven tests..."
+  export MAVEN_OPTS="-Xmx8g"
   mvn -Dparallel=classes -DthreadCount=4 -Dtest=swift.** test
 popd

.github/scripts-macosx/run-tests-cpp.sh

@@ -3,5 +3,6 @@
 set -euo pipefail
 
 pushd runtime-testsuite
+export MAVEN_OPTS="-Xmx8g"
 mvn -Dparallel=classes -DthreadCount=4 -Dtest=cpp.** test
 popd

.github/scripts-macosx/run-tests-dotnet.sh

@@ -13,5 +13,6 @@ dotnet build -c Release -f netstandard2.0 runtime/CSharp/Antlr4.csproj
 
 # run tests
 pushd runtime-testsuite/
+export MAVEN_OPTS="-Xmx8g"
 mvn -Dparallel=classes -DthreadCount=4 -Dtest=csharp.** test
 popd

.github/scripts-macosx/run-tests-swift.sh

@@ -38,6 +38,7 @@ swift build --version
 cd runtime-testsuite/
 #  mvn -e -Dparallel=classes -DthreadCount=4 -Dtest=swift.** test
 # I don't know swift enough to make it parallel. revert to single threaded
+export MAVEN_OPTS="-Xmx8g"
 mvn -e -Dtest=swift.** test
 rc=$?
 cat target/surefire-reports/*.dumpstream || true

.github/scripts-windows/run-tests-csharp.cmd

@@ -1,5 +1,6 @@
 dotnet build runtime/CSharp/src/Antlr4.csproj -c Release
 dotnet pack runtime/CSharp/src/Antlr4.csproj -c Release
 cd runtime-testsuite
+export MAVEN_OPTS="-Xmx8g"
 mvn -Dparallel=classes -DthreadCount=2 -Dtest=csharp.** test
 cd ..

.github/scripts-windows/run-tests-dart.cmd

@@ -1,5 +1,6 @@
 C:\ProgramData\chocolatey\bin\choco.exe -y install dart-sdk
 
 cd runtime-testsuite
+export MAVEN_OPTS="-Xmx8g"
 mvn -Dtest=dart.** test -Dantlr-dart-dart="C:\tools\dart-sdk\bin\dart.exe" -Dantlr-dart-pub="C:\tools\dart-sdk\bin\pub.bat" -Dantlr-dart-dart2native="C:\tools\dart-sdk\bin\dart2native.bat"
 cd ..

.github/scripts-windows/run-tests-go.cmd

@@ -1,3 +1,4 @@
 cd runtime-testsuite
+export MAVEN_OPTS="-Xmx8g"
 mvn -Dparallel=classes -DthreadCount=2 -Dtest=go.** test
 cd ..

.github/scripts-windows/run-tests-java.cmd

@@ -1,3 +1,4 @@
 cd runtime-testsuite
+export MAVEN_OPTS="-Xmx8g"
 mvn -Dparallel=classes -DthreadCount=2 -Dtest=java.** test
 cd ..

.github/scripts-windows/run-tests-javascript.cmd

@@ -1,3 +1,4 @@
 cd runtime-testsuite
+export MAVEN_OPTS="-Xmx8g"
 mvn -Dparallel=classes -DthreadCount=2 -Dtest=javascript.** test
 cd ..

.github/scripts-windows/run-tests-php.cmd

@@ -4,5 +4,6 @@ git clone https://github.com/antlr/antlr-php-runtime.git
 move antlr-php-runtime runtime\PHP
 
 cd runtime-testsuite
+export MAVEN_OPTS="-Xmx8g"
 mvn -Dparallel=classes -DthreadCount=2 -Dtest=php.** test -Dantlr-php-php="C:\tools\php81\php.exe"
 cd ..

.github/scripts-windows/run-tests-python2.cmd

@@ -1,3 +1,4 @@
 cd runtime-testsuite
+export MAVEN_OPTS="-Xmx8g"
 mvn -Dparallel=classes -DthreadCount=2 -Dantlr-python2-python="C:\Python27\python.exe" -Dtest=python2.** test
 cd ..

.github/scripts-windows/run-tests-python3.cmd

@@ -1,3 +1,4 @@
 cd runtime-testsuite
+export MAVEN_OPTS="-Xmx8g"
 mvn -Dparallel=classes -DthreadCount=2 -Dantlr-python3-python="C:\Python310\python.exe" -Dtest=python3.** test
 cd ..

runtime-testsuite/pom.xml

@@ -132,6 +132,8 @@
                 <artifactId>maven-compiler-plugin</artifactId>
                 <configuration>
                     <release>8</release>
+                    <source>9</source>
+                    <target>9</target>
                 </configuration>
             </plugin>
         </plugins>

runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/LexerExec/TokenType0xFFFF.txt

@@ -0,0 +1,16 @@
+[type]
+Lexer
+
+[grammar]
+lexer grammar L;
+T_FFFF: 'FFFF' -> type(65535);
+
+[input]
+FFFF
+
+[output]
+[@0,0:3='FFFF',<65535>,1:0]
+[@1,4:3='<EOF>',<-1>,1:4]
+
+[skip]
+Java

runtime-testsuite/resources/org/antlr/v4/test/runtime/templates/Go.test.stg

@@ -104,7 +104,7 @@ PositionAdjustingLexer() ::= <<
 func (p *PositionAdjustingLexer) NextToken() antlr.Token {
 	if _, ok := p.Interpreter.(*PositionAdjustingLexerATNSimulator); !ok {
 		lexerDeserializer := antlr.NewATNDeserializer(nil)
-		lexerAtn := lexerDeserializer.DeserializeFromUInt16(serializedLexerAtn)
+		lexerAtn := lexerDeserializer.Deserialize(serializedLexerAtn)
 		p.Interpreter = NewPositionAdjustingLexerATNSimulator(p, lexerAtn, p.Interpreter.DecisionToDFA(), p.Interpreter.SharedContextCache())
 		p.Virt = p
 	}

runtime-testsuite/test/org/antlr/v4/test/runtime/BaseRuntimeTest.java

@@ -366,9 +366,10 @@ public static RuntimeTestDescriptor[] getRuntimeTestDescriptors(String group, St
 		}
 
 		if (group.equals("LexerExec")) {
-			descriptors.add(GeneratedLexerDescriptors.getLineSeparatorLfTest(targetName));
-			descriptors.add(GeneratedLexerDescriptors.getLineSeparatorCrLfTest(targetName));
+			descriptors.add(GeneratedLexerDescriptors.getLineSeparatorLfDescriptor(targetName));
+			descriptors.add(GeneratedLexerDescriptors.getLineSeparatorCrLfDescriptor(targetName));
 			descriptors.add(GeneratedLexerDescriptors.getLargeLexerDescriptor(targetName));
+			descriptors.add(GeneratedLexerDescriptors.getAtnStatesSizeMoreThan65535Descriptor(targetName));
 		}
 
 		return descriptors.toArray(new RuntimeTestDescriptor[0]);

runtime-testsuite/test/org/antlr/v4/test/runtime/GeneratedLexerDescriptors.java

@@ -1,7 +1,9 @@
 package org.antlr.v4.test.runtime;
 
+import java.util.*;
+
 public class GeneratedLexerDescriptors {
-	static RuntimeTestDescriptor getLineSeparatorLfTest(String targetName) {
+	static RuntimeTestDescriptor getLineSeparatorLfDescriptor(String targetName) {
 		UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor();
 		result.name = "LineSeparatorLf";
 		result.targetName = targetName;
@@ -20,7 +22,7 @@ static RuntimeTestDescriptor getLineSeparatorLfTest(String targetName) {
 		return result;
 	}
 
-	static RuntimeTestDescriptor getLineSeparatorCrLfTest(String targetName) {
+	static RuntimeTestDescriptor getLineSeparatorCrLfDescriptor(String targetName) {
 		UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor();
 		result.name = "LineSeparatorCrLf";
 		result.targetName = targetName;
@@ -65,4 +67,57 @@ static RuntimeTestDescriptor getLargeLexerDescriptor(String targetName) {
 				"[@1,5:4='<EOF>',<-1>,1:5]\n";
 		return result;
 	}
+
+	static RuntimeTestDescriptor getAtnStatesSizeMoreThan65535Descriptor(String targetName) {
+		UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor();
+		result.name = "AtnStatesSizeMoreThan65535";
+		result.notes = "Regression for https://github.com/antlr/antlr4/issues/1863";
+		result.targetName = targetName;
+		result.testType = "Lexer";
+
+		final int tokensCount = 1024;
+		final String suffix = String.join("", Collections.nCopies(70, "_"));
+
+		String grammarName = "L";
+		StringBuilder grammar = new StringBuilder();
+		grammar.append("lexer grammar ").append(grammarName).append(";\n");
+		grammar.append('\n');
+		StringBuilder input = new StringBuilder();
+		StringBuilder output = new StringBuilder();
+		int startOffset;
+		int stopOffset = -2;
+		for (int i = 0; i < tokensCount; i++) {
+			String ruleName = String.format("T_%06d", i);
+			String value = ruleName+suffix;
+			grammar.append(ruleName).append(": '").append(value).append("';\n");
+			input.append(value).append('\n');
+
+			startOffset = stopOffset + 2;
+			stopOffset += value.length() + 1;
+
+			output.append("[@").append(i).append(',').append(startOffset).append(':').append(stopOffset)
+					.append("='").append(value).append("',<").append(i + 1).append(">,").append(i + 1)
+					.append(":0]\n");
+		}
+
+		grammar.append("\n");
+		grammar.append("WS: [ \\t\\r\\n]+ -> skip;\n");
+
+		startOffset = stopOffset + 2;
+		stopOffset = startOffset - 1;
+		output.append("[@").append(tokensCount).append(',').append(startOffset).append(':').append(stopOffset)
+				.append("='<EOF>',<-1>,").append(tokensCount + 1).append(":0]\n");
+
+		result.grammar = grammar.toString();
+		result.grammarName = grammarName;
+		result.input = input.toString();
+		result.output = output.toString();
+
+		List<String> all = Arrays.asList("CSharp", "Python2", "Python3", "Cpp", "Go", "PHP", "Swift", "Java", "JavaScript", "Dart");
+		result.skipTargets.addAll(all);
+//		result.skipTargets.add("Java"); // can't handle > 16bit states yet
+//		result.skipTargets.add("JavaScript"); // doesn't terminate
+//		result.skipTargets.add("Go"); // syntax error
+		return result;
+	}
 }

runtime-testsuite/test/org/antlr/v4/test/runtime/java/api/perf/TimeLexerSpeed.java

@@ -55,7 +55,7 @@
  *
  *  Sample output on OS X with 4 GHz Intel Core i7 (us == microseconds, 1/1000 of a millisecond):
  *
- Java VM args: -Xms2G -Xmx2G
+ Java VM args: -Xms2G -Xmx8g
  Warming up Java compiler....
  load_legacy_java_ascii_file average time    53us size  58384b over 3500 loads of 29038 symbols from Parser.java
  load_legacy_java_ascii_file average time    27us size  15568b over 3500 loads of  7625 symbols from RuleContext.java

runtime-testsuite/test/org/antlr/v4/test/runtime/php/BasePHPTest.java

@@ -244,7 +244,7 @@ private String locateTool(String tool) {
 			return phpPath;
 		}
 
-		String[] roots = {"/usr/local/bin/", "/opt/local/bin", "/usr/bin/"};
+		String[] roots = {"/usr/local/bin/", "/opt/local/bin", "/opt/homebrew/bin/", "/usr/bin/"};
 
 		for (String root: roots) {
 			if (new File(root + tool).exists()) {

runtime/CSharp/src/Atn/ATNDeserializer.cs

@@ -190,15 +190,7 @@ protected internal virtual void ReadLexerActions(ATN atn)
 				{
 					LexerActionType actionType = (LexerActionType)ReadInt();
 					int data1 = ReadInt();
-					if (data1 == unchecked((int)(0xFFFF)))
-					{
-						data1 = -1;
-					}
 					int data2 = ReadInt();
-					if (data2 == unchecked((int)(0xFFFF)))
-					{
-						data2 = -1;
-					}
 					ILexerAction lexerAction = LexerActionFactory(actionType, data1, data2);
 					atn.lexerActions[i_10] = lexerAction;
 				}
@@ -369,9 +361,6 @@ protected internal virtual void ReadRules(ATN atn)
 				atn.ruleToStartState[i_5] = startState;
 				if (atn.grammarType == ATNType.Lexer) {
 					int tokenType = ReadInt ();
-					if (tokenType == unchecked((int)(0xFFFF))) {
-						tokenType = TokenConstants.EOF;
-					}
 					atn.ruleToTokenType [i_5] = tokenType;
 				}
 			}

runtime/Cpp/runtime/src/Parser.cpp

@@ -38,7 +38,7 @@ struct BypassAltsAtnCache final {
   /// bypass alternatives.
   ///
   /// <seealso cref= ATNDeserializationOptions#isGenerateRuleBypassTransitions() </seealso>
-  std::map<std::vector<uint16_t>, std::unique_ptr<const atn::ATN>> map;
+  std::map<std::vector<int>, std::unique_ptr<const atn::ATN>> map;
 };
 
 BypassAltsAtnCache* getBypassAltsAtnCache() {
@@ -229,7 +229,7 @@ TokenFactory<CommonToken>* Parser::getTokenFactory() {
 
 
 const atn::ATN& Parser::getATNWithBypassAlts() {
-  const std::vector<uint16_t> &serializedAtn = getSerializedATN();
+  const std::vector<int> &serializedAtn = getSerializedATN();
   if (serializedAtn.empty()) {
     throw UnsupportedOperationException("The current parser does not support an ATN with bypass alternatives.");
   }

runtime/Cpp/runtime/src/Recognizer.h

@@ -53,7 +53,7 @@ namespace antlr4 {
     /// For interpreters, we don't know their serialized ATN despite having
     /// created the interpreter from it.
     /// </summary>
-    virtual const std::vector<uint16_t>& getSerializedATN() const {
+    virtual const std::vector<int>& getSerializedATN() const {
       throw "there is no serialized ATN";
     }