[WIP] A working prototype for a suggestions helper

.github/workflows/hosted.yml

@@ -175,7 +175,7 @@ jobs:
           typescript,
           php,
           python3,
-          # swift,
+#         swift,
         ]
         exclude:
           - os: windows-2022

runtime-testsuite/pom.xml

@@ -68,6 +68,12 @@
 		    <artifactId>jol-core</artifactId>
 		    <version>0.16</version>
 		</dependency>
+        <dependency>
+            <groupId>org.yaml</groupId>
+            <artifactId>snakeyaml</artifactId>
+            <version>2.2</version>
+            <scope>test</scope>
+        </dependency>
 	</dependencies>
 
 	<build>

runtime-testsuite/resources/org/antlr/v4/test/runtime/expected-tokens/java-class-access-modifier.yml

@@ -0,0 +1,17 @@
+input: "abstract "
+caret:
+  line: 1
+  column: 9
+expected:
+  - final
+  - protected
+  - static
+  - "@"
+  - enum
+  - public
+  - ;
+  - class
+  - abstract
+  - interface
+  - private
+  - strictfp

runtime-testsuite/resources/org/antlr/v4/test/runtime/expected-tokens/java-class-class.yml

@@ -0,0 +1,6 @@
+input: "class "
+caret:
+  line: 1
+  column: 6
+expected:
+  - Identifier

runtime-testsuite/resources/org/antlr/v4/test/runtime/expected-tokens/java-member-access-modifier.yml

@@ -0,0 +1,31 @@
+input: "public class Foo { public static    "
+caret:
+  line: 1
+  column: 34
+expected:
+  - final
+  - protected
+  - static
+  - "@"
+  - enum
+  - public
+  - class
+  - abstract
+  - interface
+  - private
+  - strictfp
+  - boolean
+  - byte
+  - char
+  - double
+  - float
+  - int
+  - long
+  - native
+  - short
+  - synchronized
+  - transient
+  - void
+  - volatile
+  - <
+  - Identifier

runtime-testsuite/resources/org/antlr/v4/test/runtime/expected-tokens/java-start.yml

@@ -0,0 +1,19 @@
+caret:
+  line: 1
+  column: 0
+expected:
+  - import
+  - final
+  - package
+  - protected
+  - static
+  - "@"
+  - enum
+  - public
+  - ;
+  - class
+  - abstract
+  - interface
+  - private
+  - EOF
+  - strictfp

runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/package_ts.json

@@ -2,8 +2,5 @@
   "type": "module",
    "devDependencies": {
     "@types/node": "^18.0.5"
-   },
-  "dependencies": {
-    "antlr4": "^4.13.1"
-  }
+   }
 }

runtime-testsuite/test/org/antlr/v4/runtime/TestSuggestionHelper.java

@@ -0,0 +1,98 @@
+package org.antlr.v4.runtime;
+
+import org.antlr.v4.runtime.misc.IntervalSet;
+import org.antlr.v4.runtime.misc.Pair;
+import org.antlr.v4.test.runtime.java.api.JavaLexer;
+import org.antlr.v4.test.runtime.java.api.JavaParser;
+import org.junit.jupiter.api.Test;
+import org.opentest4j.AssertionFailedError;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TestSuggestionHelper {
+
+	@Test
+	public void matchesEachExpectation() throws Exception {
+		List<Pair<Path, Error>> errors = new ArrayList<>();
+		URL url = this.getClass().getResource("/org/antlr/v4/test/runtime/expected-tokens/");
+		try(Stream<Path> walk = Files.walk(Paths.get(url.toURI()), 1)) {
+			for (Iterator<Path> it = walk.iterator(); it.hasNext(); ) {
+				Path path = it.next();
+				if(path.toString().endsWith(".yml")) try {
+					matchesExpectation(path);
+				} catch(AssertionFailedError e) {
+					errors.add(new Pair<>(path, e));
+				}
+			}
+		}
+		// fail the test if required
+		if(!errors.isEmpty()) {
+			errors.forEach(p -> System.err.println("Failed: " + p.a.toString()));
+			throw errors.get(0).b;
+		}
+	}
+
+	@Test
+	public void matchesOneExpectation() throws Exception {
+		URL url = this.getClass().getResource("/org/antlr/v4/test/runtime/expected-tokens/");
+		Path parent = Paths.get(url.toURI());
+		File file = new File(parent.toFile(), "java-start.yml");
+		matchesExpectation(file.toPath());
+	}
+
+	static class Expectation {
+		public String input;
+		public int line;
+		public int column;
+		public Set<String> expectedTokens;
+
+	}
+	void matchesExpectation(Path expectations) throws Exception {
+		Expectation expectation = parseExpectation(expectations);
+		CharStream input = CharStreams.fromString(expectation.input);
+		Lexer lexer = new JavaLexer(input);
+		lexer.setTokenFactory(CommonTokenWithStatesFactory.DEFAULT);
+		TokenStream stream = new CommonTokenStream(lexer);
+		JavaParser parser = new JavaParser(stream);
+		parser.setErrorHandler(new RecordingErrorStrategy());
+		RuleContext context = parser.compilationUnit();
+		Token token = stream.lastTokenAt(expectation.line, expectation.column);
+		Pair<RuleContext, IntervalSet> contextsAndIntervals = parser.getExpectedTokensAt(context, (TokenWithStates)token);
+		Set<String> actual = contextsAndIntervals.b.toSet().stream()
+			.map(t -> parser.getVocabulary().getDisplayName(t))
+			.map(s -> s.startsWith("'") ? s.substring(1, s.length() - 1) : s)
+			.collect(Collectors.toSet());
+		assertEquals(expectation.expectedTokens, actual);
+	}
+
+	static Expectation parseExpectation(Path expectationsFile) throws Exception {
+		Yaml yaml = new Yaml();
+		try(InputStream input = Files.newInputStream(expectationsFile)) {
+			Map<String, Object> doc = yaml.load(input);
+			Expectation expectation = new Expectation();
+			expectation.input = doc.getOrDefault("input", "").toString();
+			Object value = doc.get("caret");
+			if(value instanceof Map) {
+				expectation.line = (int) ((Map<String, Object>) value).get("line");
+				expectation.column = (int) ((Map<String, Object>) value).get("column");
+			}
+			value = doc.get("expected");
+			if(value instanceof List) {
+				expectation.expectedTokens = new HashSet<>(((List<String>) value));
+			}
+			return expectation;
+		}
+	}
+
+}

runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeRunner.java

@@ -189,9 +189,11 @@ public State run(RunOptions runOptions) {
 			return generatedState;
 		}
 
-		if (!initAntlrRuntimeIfRequired(runOptions)) {
+		InitializationStatus initStatus = initAntlrRuntimeIfRequired(runOptions);
+
+		if (!initStatus.isInitialized) {
 			// Do not repeat ANTLR runtime initialization error
-			return new CompiledState(generatedState, new Exception(getTitleName() + " ANTLR runtime is not initialized"));
+			return new CompiledState(generatedState, new Exception(getTitleName() + " ANTLR runtime is not initialized", initStatus.exception));
 		}
 
 		writeRecognizerFile(runOptions);
@@ -261,7 +263,7 @@ protected String grammarParseRuleToRecognizerName(String startRuleName) {
 	protected void addExtraRecognizerParameters(ST template) {
 	}
 
-	private boolean initAntlrRuntimeIfRequired(RunOptions runOptions) {
+	private InitializationStatus initAntlrRuntimeIfRequired(RunOptions runOptions) {
 		String language = getLanguage();
 		InitializationStatus status;
 
@@ -275,7 +277,7 @@ private boolean initAntlrRuntimeIfRequired(RunOptions runOptions) {
 		}
 
 		if (status.isInitialized != null) {
-			return status.isInitialized;
+			return status;
 		}
 
 		// Locking per runtime, several runtimes can be being initialized simultaneously
@@ -292,7 +294,7 @@ private boolean initAntlrRuntimeIfRequired(RunOptions runOptions) {
 				status.exception = exception;
 			}
 		}
-		return status.isInitialized;
+		return status;
 	}
 
 	protected void initRuntime(RunOptions runOptions) throws Exception {

runtime/Java/src/org/antlr/v4/runtime/CommonTokenWithStates.java

@@ -0,0 +1,36 @@
+package org.antlr.v4.runtime;
+
+import org.antlr.v4.runtime.atn.ATNState;
+import org.antlr.v4.runtime.misc.Pair;
+
+public class CommonTokenWithStates extends CommonToken implements TokenWithStates {
+
+	int _previousState = ATNState.INVALID_STATE_NUMBER;
+	int _followState = ATNState.INVALID_STATE_NUMBER;
+
+	public CommonTokenWithStates(Pair<TokenSource, CharStream> source, int type, int channel, int start, int stop) {
+		super(source, type, channel, start, stop);
+	}
+
+	public CommonTokenWithStates(int type, String text) {
+		super(type, text);
+	}
+
+	@Override
+	public void setPreviousState(int state) {
+		this._previousState = state;
+	}
+
+	@Override
+	public int getPreviousState() {
+		return _previousState;
+	}
+
+	public void setFollowState(int state) {
+		this._followState = state;
+	}
+
+	public int getFollowState() {
+		return _followState;
+	}
+}

runtime/Java/src/org/antlr/v4/runtime/CommonTokenWithStatesFactory.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+package org.antlr.v4.runtime;
+
+import org.antlr.v4.runtime.misc.Interval;
+import org.antlr.v4.runtime.misc.Pair;
+
+/**
+ * This implementation of {@link TokenFactory} creates
+ * {@link CommonTokenWithStates} objects.
+ */
+public class CommonTokenWithStatesFactory implements TokenFactory<CommonTokenWithStates> {
+
+	/**
+	 * The default {@link CommonTokenWithStatesFactory} instance.
+	 *
+	 * <p>
+	 * This token factory does not explicitly copy token text when constructing
+	 * tokens.</p>
+	 */
+	public static final TokenFactory<CommonTokenWithStates> DEFAULT = new CommonTokenWithStatesFactory();
+
+	/**
+	 * Indicates whether {@link CommonToken#setText} should be called after
+	 * constructing tokens to explicitly set the text. This is useful for cases
+	 * where the input stream might not be able to provide arbitrary substrings
+	 * of text from the input after the lexer creates a token (e.g. the
+	 * implementation of {@link CharStream#getText} in
+	 * {@link UnbufferedCharStream} throws an
+	 * {@link UnsupportedOperationException}). Explicitly setting the token text
+	 * allows {@link Token#getText} to be called at any time regardless of the
+	 * input stream implementation.
+	 *
+	 * <p>
+	 * The default value is {@code false} to avoid the performance and memory
+	 * overhead of copying text for every token unless explicitly requested.</p>
+	 */
+	protected final boolean copyText;
+
+	/**
+	 * Constructs a {@link CommonTokenWithStatesFactory} with the specified value for
+	 * {@link #copyText}.
+	 *
+	 * <p>
+	 * When {@code copyText} is {@code false}, the {@link #DEFAULT} instance
+	 * should be used instead of constructing a new instance.</p>
+	 *
+	 * @param copyText The value for {@link #copyText}.
+	 */
+	public CommonTokenWithStatesFactory(boolean copyText) { this.copyText = copyText; }
+
+	/**
+	 * Constructs a {@link CommonTokenWithStatesFactory} with {@link #copyText} set to
+	 * {@code false}.
+	 *
+	 * <p>
+	 * The {@link #DEFAULT} instance should be used instead of calling this
+	 * directly.</p>
+	 */
+	public CommonTokenWithStatesFactory() { this(false); }
+
+	@Override
+	public CommonTokenWithStates create(Pair<TokenSource, CharStream> source, int type, String text,
+										int channel, int start, int stop,
+										int line, int charPositionInLine)
+	{
+		CommonTokenWithStates t = new CommonTokenWithStates(source, type, channel, start, stop);
+		t.setLine(line);
+		t.setCharPositionInLine(charPositionInLine);
+		if ( text!=null ) {
+			t.setText(text);
+		}
+		else if ( copyText && source.b != null ) {
+			t.setText(source.b.getText(Interval.of(start,stop)));
+		}
+
+		return t;
+	}
+
+	@Override
+	public CommonTokenWithStates create(int type, String text) {
+		return new CommonTokenWithStates(type, text);
+	}
+}

runtime/Java/src/org/antlr/v4/runtime/Lexer.java

@@ -35,7 +35,7 @@ public abstract class Lexer extends Recognizer<Integer, LexerATNSimulator>
 	protected Pair<TokenSource, CharStream> _tokenFactorySourcePair;
 
 	/** How to create token objects */
-	protected TokenFactory<?> _factory = CommonTokenFactory.DEFAULT;
+	protected TokenFactory<? extends Token> _factory = CommonTokenFactory.DEFAULT;
 
 	/** The goal of all lexer rules/methods is to create a token object.
 	 *  This is an instance variable as multiple rules may collaborate to