Skip to content

Commit

Permalink
Have scanJsDocToken scan keywords (#27162)
Browse files Browse the repository at this point in the history 
* Have scanJsDocToken scan keywords

* Update API
  • Loading branch information
Andy committed Oct 22, 2018
1 parent fc82c4d commit 2ef3c01
Show file tree
Hide file tree
Showing 16 changed files with 213 additions and 111 deletions.
19 changes: 9 additions & 10 deletions src/compiler/parser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6517,7 +6517,7 @@ namespace ts {
}
}

function skipWhitespaceOrAsterisk(next: () => void): void {
function skipWhitespaceOrAsterisk(): void {
if (token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia) {
if (lookAhead(isNextNonwhitespaceTokenEndOfFile)) {
return; // Don't skip whitespace prior to EoF (or end of comment) - that shouldn't be included in any node's range
Expand All @@ -6532,7 +6532,7 @@ namespace ts {
else if (token() === SyntaxKind.AsteriskToken) {
precedingLineBreak = false;
}
next();
nextJSDocToken();
}
}

Expand All @@ -6542,9 +6542,8 @@ namespace ts {
atToken.end = scanner.getTextPos();
nextJSDocToken();

// Use 'nextToken' instead of 'nextJsDocToken' so we can parse a type like 'number' in `@enum number`
const tagName = parseJSDocIdentifierName(/*message*/ undefined, nextToken);
skipWhitespaceOrAsterisk(nextToken);
const tagName = parseJSDocIdentifierName(/*message*/ undefined);
skipWhitespaceOrAsterisk();

let tag: JSDocTag | undefined;
switch (tagName.escapedText) {
Expand Down Expand Up @@ -6688,7 +6687,7 @@ namespace ts {
}

function tryParseTypeExpression(): JSDocTypeExpression | undefined {
skipWhitespaceOrAsterisk(nextJSDocToken);
skipWhitespaceOrAsterisk();
return token() === SyntaxKind.OpenBraceToken ? parseJSDocTypeExpression() : undefined;
}

Expand Down Expand Up @@ -6728,7 +6727,7 @@ namespace ts {
function parseParameterOrPropertyTag(atToken: AtToken, tagName: Identifier, target: PropertyLikeParse, indent: number): JSDocParameterTag | JSDocPropertyTag {
let typeExpression = tryParseTypeExpression();
let isNameFirst = !typeExpression;
skipWhitespaceOrAsterisk(nextJSDocToken);
skipWhitespaceOrAsterisk();

const { name, isBracketed } = parseBracketNameInPropertyAndParamTag();
skipWhitespace();
Expand Down Expand Up @@ -6862,7 +6861,7 @@ namespace ts {

function parseTypedefTag(atToken: AtToken, tagName: Identifier, indent: number): JSDocTypedefTag {
const typeExpression = tryParseTypeExpression();
skipWhitespaceOrAsterisk(nextJSDocToken);
skipWhitespaceOrAsterisk();

const typedefTag = <JSDocTypedefTag>createNode(SyntaxKind.JSDocTypedefTag, atToken.pos);
typedefTag.atToken = atToken;
Expand Down Expand Up @@ -7115,7 +7114,7 @@ namespace ts {
return entity;
}

function parseJSDocIdentifierName(message?: DiagnosticMessage, next: () => void = nextJSDocToken): Identifier {
function parseJSDocIdentifierName(message?: DiagnosticMessage): Identifier {
if (!tokenIsIdentifierOrKeyword(token())) {
return createMissingNode<Identifier>(SyntaxKind.Identifier, /*reportAtCurrentPosition*/ !message, message || Diagnostics.Identifier_expected);
}
Expand All @@ -7126,7 +7125,7 @@ namespace ts {
result.escapedText = escapeLeadingUnderscores(scanner.getTokenText());
finishNode(result, end);

next();
nextJSDocToken();
return result;
}
}
Expand Down
166 changes: 86 additions & 80 deletions src/compiler/scanner.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,81 +60,87 @@ namespace ts {
tryScan<T>(callback: () => T): T;
}

const textToToken = createMapFromTemplate({
"abstract": SyntaxKind.AbstractKeyword,
"any": SyntaxKind.AnyKeyword,
"as": SyntaxKind.AsKeyword,
"boolean": SyntaxKind.BooleanKeyword,
"break": SyntaxKind.BreakKeyword,
"case": SyntaxKind.CaseKeyword,
"catch": SyntaxKind.CatchKeyword,
"class": SyntaxKind.ClassKeyword,
"continue": SyntaxKind.ContinueKeyword,
"const": SyntaxKind.ConstKeyword,
"constructor": SyntaxKind.ConstructorKeyword,
"debugger": SyntaxKind.DebuggerKeyword,
"declare": SyntaxKind.DeclareKeyword,
"default": SyntaxKind.DefaultKeyword,
"delete": SyntaxKind.DeleteKeyword,
"do": SyntaxKind.DoKeyword,
"else": SyntaxKind.ElseKeyword,
"enum": SyntaxKind.EnumKeyword,
"export": SyntaxKind.ExportKeyword,
"extends": SyntaxKind.ExtendsKeyword,
"false": SyntaxKind.FalseKeyword,
"finally": SyntaxKind.FinallyKeyword,
"for": SyntaxKind.ForKeyword,
"from": SyntaxKind.FromKeyword,
"function": SyntaxKind.FunctionKeyword,
"get": SyntaxKind.GetKeyword,
"if": SyntaxKind.IfKeyword,
"implements": SyntaxKind.ImplementsKeyword,
"import": SyntaxKind.ImportKeyword,
"in": SyntaxKind.InKeyword,
"infer": SyntaxKind.InferKeyword,
"instanceof": SyntaxKind.InstanceOfKeyword,
"interface": SyntaxKind.InterfaceKeyword,
"is": SyntaxKind.IsKeyword,
"keyof": SyntaxKind.KeyOfKeyword,
"let": SyntaxKind.LetKeyword,
"module": SyntaxKind.ModuleKeyword,
"namespace": SyntaxKind.NamespaceKeyword,
"never": SyntaxKind.NeverKeyword,
"new": SyntaxKind.NewKeyword,
"null": SyntaxKind.NullKeyword,
"number": SyntaxKind.NumberKeyword,
"object": SyntaxKind.ObjectKeyword,
"package": SyntaxKind.PackageKeyword,
"private": SyntaxKind.PrivateKeyword,
"protected": SyntaxKind.ProtectedKeyword,
"public": SyntaxKind.PublicKeyword,
"readonly": SyntaxKind.ReadonlyKeyword,
"require": SyntaxKind.RequireKeyword,
"global": SyntaxKind.GlobalKeyword,
"return": SyntaxKind.ReturnKeyword,
"set": SyntaxKind.SetKeyword,
"static": SyntaxKind.StaticKeyword,
"string": SyntaxKind.StringKeyword,
"super": SyntaxKind.SuperKeyword,
"switch": SyntaxKind.SwitchKeyword,
"symbol": SyntaxKind.SymbolKeyword,
"this": SyntaxKind.ThisKeyword,
"throw": SyntaxKind.ThrowKeyword,
"true": SyntaxKind.TrueKeyword,
"try": SyntaxKind.TryKeyword,
"type": SyntaxKind.TypeKeyword,
"typeof": SyntaxKind.TypeOfKeyword,
"undefined": SyntaxKind.UndefinedKeyword,
"unique": SyntaxKind.UniqueKeyword,
"unknown": SyntaxKind.UnknownKeyword,
"var": SyntaxKind.VarKeyword,
"void": SyntaxKind.VoidKeyword,
"while": SyntaxKind.WhileKeyword,
"with": SyntaxKind.WithKeyword,
"yield": SyntaxKind.YieldKeyword,
"async": SyntaxKind.AsyncKeyword,
"await": SyntaxKind.AwaitKeyword,
"of": SyntaxKind.OfKeyword,
const textToKeywordObj: MapLike<KeywordSyntaxKind> = {
abstract: SyntaxKind.AbstractKeyword,
any: SyntaxKind.AnyKeyword,
as: SyntaxKind.AsKeyword,
boolean: SyntaxKind.BooleanKeyword,
break: SyntaxKind.BreakKeyword,
case: SyntaxKind.CaseKeyword,
catch: SyntaxKind.CatchKeyword,
class: SyntaxKind.ClassKeyword,
continue: SyntaxKind.ContinueKeyword,
const: SyntaxKind.ConstKeyword,
["" + "constructor"]: SyntaxKind.ConstructorKeyword,
debugger: SyntaxKind.DebuggerKeyword,
declare: SyntaxKind.DeclareKeyword,
default: SyntaxKind.DefaultKeyword,
delete: SyntaxKind.DeleteKeyword,
do: SyntaxKind.DoKeyword,
else: SyntaxKind.ElseKeyword,
enum: SyntaxKind.EnumKeyword,
export: SyntaxKind.ExportKeyword,
extends: SyntaxKind.ExtendsKeyword,
false: SyntaxKind.FalseKeyword,
finally: SyntaxKind.FinallyKeyword,
for: SyntaxKind.ForKeyword,
from: SyntaxKind.FromKeyword,
function: SyntaxKind.FunctionKeyword,
get: SyntaxKind.GetKeyword,
if: SyntaxKind.IfKeyword,
implements: SyntaxKind.ImplementsKeyword,
import: SyntaxKind.ImportKeyword,
in: SyntaxKind.InKeyword,
infer: SyntaxKind.InferKeyword,
instanceof: SyntaxKind.InstanceOfKeyword,
interface: SyntaxKind.InterfaceKeyword,
is: SyntaxKind.IsKeyword,
keyof: SyntaxKind.KeyOfKeyword,
let: SyntaxKind.LetKeyword,
module: SyntaxKind.ModuleKeyword,
namespace: SyntaxKind.NamespaceKeyword,
never: SyntaxKind.NeverKeyword,
new: SyntaxKind.NewKeyword,
null: SyntaxKind.NullKeyword,
number: SyntaxKind.NumberKeyword,
object: SyntaxKind.ObjectKeyword,
package: SyntaxKind.PackageKeyword,
private: SyntaxKind.PrivateKeyword,
protected: SyntaxKind.ProtectedKeyword,
public: SyntaxKind.PublicKeyword,
readonly: SyntaxKind.ReadonlyKeyword,
require: SyntaxKind.RequireKeyword,
global: SyntaxKind.GlobalKeyword,
return: SyntaxKind.ReturnKeyword,
set: SyntaxKind.SetKeyword,
static: SyntaxKind.StaticKeyword,
string: SyntaxKind.StringKeyword,
super: SyntaxKind.SuperKeyword,
switch: SyntaxKind.SwitchKeyword,
symbol: SyntaxKind.SymbolKeyword,
this: SyntaxKind.ThisKeyword,
throw: SyntaxKind.ThrowKeyword,
true: SyntaxKind.TrueKeyword,
try: SyntaxKind.TryKeyword,
type: SyntaxKind.TypeKeyword,
typeof: SyntaxKind.TypeOfKeyword,
undefined: SyntaxKind.UndefinedKeyword,
unique: SyntaxKind.UniqueKeyword,
unknown: SyntaxKind.UnknownKeyword,
var: SyntaxKind.VarKeyword,
void: SyntaxKind.VoidKeyword,
while: SyntaxKind.WhileKeyword,
with: SyntaxKind.WithKeyword,
yield: SyntaxKind.YieldKeyword,
async: SyntaxKind.AsyncKeyword,
await: SyntaxKind.AwaitKeyword,
of: SyntaxKind.OfKeyword,
};

const textToKeyword = createMapFromTemplate(textToKeywordObj);

const textToToken = createMapFromTemplate<SyntaxKind>({
...textToKeywordObj,
"{": SyntaxKind.OpenBraceToken,
"}": SyntaxKind.CloseBraceToken,
"(": SyntaxKind.OpenParenToken,
Expand Down Expand Up @@ -1288,15 +1294,15 @@ namespace ts {
return result;
}

function getIdentifierToken(): SyntaxKind {
function getIdentifierToken(): SyntaxKind.Identifier | KeywordSyntaxKind {
// Reserved words are between 2 and 11 characters long and start with a lowercase letter
const len = tokenValue.length;
if (len >= 2 && len <= 11) {
const ch = tokenValue.charCodeAt(0);
if (ch >= CharacterCodes.a && ch <= CharacterCodes.z) {
token = textToToken.get(tokenValue)!;
if (token !== undefined) {
return token;
const keyword = textToKeyword.get(tokenValue);
if (keyword !== undefined) {
return token = keyword;
}
}
}
Expand Down Expand Up @@ -2016,7 +2022,7 @@ namespace ts {
pos++;
}
tokenValue = text.substring(tokenPos, pos);
return token = SyntaxKind.Identifier;
return token = getIdentifierToken();
}
else {
return token = SyntaxKind.Unknown;
Expand Down
79 changes: 78 additions & 1 deletion src/compiler/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,84 @@ namespace ts {
| SyntaxKind.DotToken
| SyntaxKind.Identifier
| SyntaxKind.NoSubstitutionTemplateLiteral
| SyntaxKind.Unknown;
| SyntaxKind.Unknown
| KeywordSyntaxKind;

export type KeywordSyntaxKind =
| SyntaxKind.AbstractKeyword
| SyntaxKind.AnyKeyword
| SyntaxKind.AsKeyword
| SyntaxKind.BooleanKeyword
| SyntaxKind.BreakKeyword
| SyntaxKind.CaseKeyword
| SyntaxKind.CatchKeyword
| SyntaxKind.ClassKeyword
| SyntaxKind.ContinueKeyword
| SyntaxKind.ConstKeyword
| SyntaxKind.ConstructorKeyword
| SyntaxKind.DebuggerKeyword
| SyntaxKind.DeclareKeyword
| SyntaxKind.DefaultKeyword
| SyntaxKind.DeleteKeyword
| SyntaxKind.DoKeyword
| SyntaxKind.ElseKeyword
| SyntaxKind.EnumKeyword
| SyntaxKind.ExportKeyword
| SyntaxKind.ExtendsKeyword
| SyntaxKind.FalseKeyword
| SyntaxKind.FinallyKeyword
| SyntaxKind.ForKeyword
| SyntaxKind.FromKeyword
| SyntaxKind.FunctionKeyword
| SyntaxKind.GetKeyword
| SyntaxKind.IfKeyword
| SyntaxKind.ImplementsKeyword
| SyntaxKind.ImportKeyword
| SyntaxKind.InKeyword
| SyntaxKind.InferKeyword
| SyntaxKind.InstanceOfKeyword
| SyntaxKind.InterfaceKeyword
| SyntaxKind.IsKeyword
| SyntaxKind.KeyOfKeyword
| SyntaxKind.LetKeyword
| SyntaxKind.ModuleKeyword
| SyntaxKind.NamespaceKeyword
| SyntaxKind.NeverKeyword
| SyntaxKind.NewKeyword
| SyntaxKind.NullKeyword
| SyntaxKind.NumberKeyword
| SyntaxKind.ObjectKeyword
| SyntaxKind.PackageKeyword
| SyntaxKind.PrivateKeyword
| SyntaxKind.ProtectedKeyword
| SyntaxKind.PublicKeyword
| SyntaxKind.ReadonlyKeyword
| SyntaxKind.RequireKeyword
| SyntaxKind.GlobalKeyword
| SyntaxKind.ReturnKeyword
| SyntaxKind.SetKeyword
| SyntaxKind.StaticKeyword
| SyntaxKind.StringKeyword
| SyntaxKind.SuperKeyword
| SyntaxKind.SwitchKeyword
| SyntaxKind.SymbolKeyword
| SyntaxKind.ThisKeyword
| SyntaxKind.ThrowKeyword
| SyntaxKind.TrueKeyword
| SyntaxKind.TryKeyword
| SyntaxKind.TypeKeyword
| SyntaxKind.TypeOfKeyword
| SyntaxKind.UndefinedKeyword
| SyntaxKind.UniqueKeyword
| SyntaxKind.UnknownKeyword
| SyntaxKind.VarKeyword
| SyntaxKind.VoidKeyword
| SyntaxKind.WhileKeyword
| SyntaxKind.WithKeyword
| SyntaxKind.YieldKeyword
| SyntaxKind.AsyncKeyword
| SyntaxKind.AwaitKeyword
| SyntaxKind.OfKeyword;

export type JsxTokenSyntaxKind =
| SyntaxKind.LessThanSlashToken
Expand Down
4 changes: 2 additions & 2 deletions tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.@link tags.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"0": {
"kind": "JSDocTag",
"pos": 63,
"end": 67,
"end": 68,
"atToken": {
"kind": "AtToken",
"pos": 63,
Expand All @@ -22,7 +22,7 @@
},
"length": 1,
"pos": 63,
"end": 67
"end": 68
},
"comment": "{@link first link}\nInside {@link link text} thing"
}
4 changes: 2 additions & 2 deletions tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
"typeParameters": {
"0": {
"kind": "TypeParameter",
"pos": 17,
"pos": 18,
"end": 19,
"name": {
"kind": "Identifier",
Expand All @@ -31,7 +31,7 @@
}
},
"length": 1,
"pos": 17,
"pos": 18,
"end": 19
}
},
Expand Down

0 comments on commit 2ef3c01

Please sign in to comment.