Skip to content

Commit

Permalink
[[FIX]] Improve handling of tab indentation.
Browse files Browse the repository at this point in the history 
By tracking the column number both with and without soft tabs,
we enable better messages for errors and warnings. This is especially
useful for editor linting integrations.

Fixed jshint#3151
  • Loading branch information
@TzviPM
TzviPM committed Apr 26, 2018
1 parent e0b4e91 commit 07513d5
Show file tree
Hide file tree
Showing 5 changed files with 138 additions and 99 deletions.
103 changes: 61 additions & 42 deletions src/lex.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,10 @@ function Lexer(source) {

this.line = 0;
this.char = 1;
this.originalChar = 1;
this.from = 1;
this.input = "";
this.originalInput = "";
this.inComment = false;
this.context = [];
this.templateStarts = [];
Expand Down Expand Up @@ -164,17 +166,24 @@ Lexer.prototype = {
* Return the next i character without actually moving the
* char pointer.
*/
peek: function(i) {
return this.input.charAt(i || 0);
peek: function(i, str) {
str = str || this.input;
return str.charAt(i || 0);
},

/*
* Move the char pointer forward i times.
*/
skip: function(i) {
i = i || 1;
this.char += i;
this.input = this.input.slice(i);
var tabExpandedI = i;
if (i === 1 && this.peek(0, this.originalInput) === '\t') {
tabExpandedI = state.tab.length;
}
this.char += tabExpandedI;
this.originalChar += i;
this.input = this.input.slice(tabExpandedI);
this.originalInput = this.originalInput.slice(i);
},

/*
Expand Down Expand Up @@ -384,7 +393,7 @@ Lexer.prototype = {
var ch2 = this.peek(1);
var rest = this.input.substr(2);
var startLine = this.line;
var startChar = this.char;
var startChar = this.originalChar;
var self = this;

// Create a comment token object and make sure it
Expand Down Expand Up @@ -770,7 +779,7 @@ Lexer.prototype = {
{
code: "W119",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ "Octal integer literal", "6" ]
},
checks,
Expand All @@ -793,7 +802,7 @@ Lexer.prototype = {
{
code: "W119",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ "Binary integer literal", "6" ]
},
checks,
Expand Down Expand Up @@ -936,7 +945,7 @@ Lexer.prototype = {
this.triggerAsync("warning", {
code: "W114",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ "\\'" ]
}, checks, function() {return state.jsonMode; });
break;
Expand Down Expand Up @@ -964,7 +973,7 @@ Lexer.prototype = {
this.triggerAsync("warning", {
code: "W115",
line: this.line,
character: this.char
character: this.originalChar
}, checks,
function() { return n >= 0 && n <= 7 && state.isStrict(); });
break;
Expand All @@ -979,7 +988,7 @@ Lexer.prototype = {
this.triggerAsync("warning", {
code: "W115",
line: this.line,
character: this.char
character: this.originalChar
}, checks,
function() { return state.isStrict(); });
break;
Expand All @@ -992,7 +1001,7 @@ Lexer.prototype = {
this.trigger("warning", {
code: "W052",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ "u" + sequence ]
});
}
Expand All @@ -1003,7 +1012,7 @@ Lexer.prototype = {
this.triggerAsync("warning", {
code: "W114",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ "\\v" ]
}, checks, function() { return state.jsonMode; });

Expand All @@ -1015,7 +1024,7 @@ Lexer.prototype = {
this.triggerAsync("warning", {
code: "W114",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ "\\x-" ]
}, checks, function() { return state.jsonMode; });

Expand Down Expand Up @@ -1050,7 +1059,7 @@ Lexer.prototype = {
var value = "";
var ch;
var startLine = this.line;
var startChar = this.char;
var startChar = this.originalChar;
var depth = this.templateStarts.length;

if (this.peek() === "`") {
Expand All @@ -1060,7 +1069,7 @@ Lexer.prototype = {
{
code: "W119",
line: this.line,
character: this.char,
character: this.originalChar,
data: ["template literal syntax", "6"]
},
checks,
Expand All @@ -1069,7 +1078,11 @@ Lexer.prototype = {
}
// Template must start with a backtick.
tokenType = Token.TemplateHead;
this.templateStarts.push({ line: this.line, char: this.char });
this.templateStarts.push({
line: this.line,
char: this.char,
originalChar: this.originalChar
});
depth = this.templateStarts.length;
this.skip(1);
this.pushContext(Context.Template);
Expand All @@ -1090,7 +1103,7 @@ Lexer.prototype = {
this.trigger("error", {
code: "E052",
line: startPos.line,
character: startPos.char
character: startPos.originalChar
});
return {
type: tokenType,
Expand Down Expand Up @@ -1167,12 +1180,12 @@ Lexer.prototype = {
this.triggerAsync("warning", {
code: "W108",
line: this.line,
character: this.char // +1?
character: this.originalChar // +1?
}, checks, function() { return state.jsonMode && quote !== "\""; });

var value = "";
var startLine = this.line;
var startChar = this.char;
var startChar = this.originalChar;
var allowNewLine = false;

this.skip();
Expand All @@ -1193,7 +1206,7 @@ Lexer.prototype = {
this.trigger("warning", {
code: "W112",
line: this.line,
character: this.char
character: this.originalChar
});
} else {
allowNewLine = false;
Expand All @@ -1204,13 +1217,13 @@ Lexer.prototype = {
this.triggerAsync("warning", {
code: "W043",
line: this.line,
character: this.char
character: this.originalChar
}, checks, function() { return !state.option.multistr; });

this.triggerAsync("warning", {
code: "W042",
line: this.line,
character: this.char
character: this.originalChar
}, checks, function() { return state.jsonMode && state.option.multistr; });
}

Expand Down Expand Up @@ -1242,7 +1255,7 @@ Lexer.prototype = {
{
code: "W113",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ "<non-printable>" ]
},
checks,
Expand Down Expand Up @@ -1310,7 +1323,7 @@ Lexer.prototype = {
{
code: "W048",
line: this.line,
character: this.char
character: this.originalChar
},
checks,
function() { return true; }
Expand All @@ -1325,7 +1338,7 @@ Lexer.prototype = {
{
code: "W049",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ char ]
},
checks,
Expand Down Expand Up @@ -1437,7 +1450,7 @@ Lexer.prototype = {
{
code: "W119",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ "Sticky RegExp flag", "6" ]
},
checks,
Expand Down Expand Up @@ -1472,7 +1485,7 @@ Lexer.prototype = {
this.trigger("error", {
code: "E016",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ malformedDesc ]
});
}
Expand All @@ -1489,24 +1502,26 @@ Lexer.prototype = {
* can be mistakenly typed on OS X with option-space. Non UTF-8 web
* pages with non-breaking pages produce syntax errors.
*/
scanNonBreakingSpaces: function() {
scanNonBreakingSpaces: function(str) {
str = str || this.input;
return state.option.nonbsp ?
this.input.search(/(\u00A0)/) : -1;
str.search(/(\u00A0)/) : -1;
},

/*
* Scan for characters that get silently deleted by one or more browsers.
*/
scanUnsafeChars: function() {
return this.input.search(reg.unsafeChars);
scanUnsafeChars: function(str) {
str = str || this.input;
return str.search(reg.unsafeChars);
},

/*
* Produce the next raw token or return 'null' if no tokens can be matched.
* This method skips over all space characters.
*/
next: function(checks) {
this.from = this.char;
this.from = this.originalChar;

// Move to the next non-space character.
while (reg.whitespace.test(this.peek())) {
Expand Down Expand Up @@ -1549,15 +1564,17 @@ Lexer.prototype = {
* switched, this method also checks for other minor warnings.
*/
nextLine: function(checks) {
var char;
var char, originalChar;

if (this.line >= this.getLines().length) {
return false;
}

this.input = this.getLines()[this.line];
this.originalInput = this.getLines()[this.line];
this.line += 1;
this.char = 1;
this.originalChar = 1;
this.from = 1;

var inputTrimmed = this.input.trim();
Expand All @@ -1583,22 +1600,24 @@ Lexer.prototype = {
}

char = this.scanNonBreakingSpaces();
originalChar = this.scanNonBreakingSpaces(this.originalInput);
if (char >= 0) {
this.triggerAsync(
"warning",
{ code: "W125", line: this.line, character: char + 1 },
{ code: "W125", line: this.line, character: originalChar + 1 },
checks,
function() { return true; }
);
}

this.input = this.input.replace(/\t/g, state.tab);
char = this.scanUnsafeChars();
originalChar = this.scanUnsafeChars(this.originalInput);

if (char >= 0) {
this.triggerAsync(
"warning",
{ code: "W100", line: this.line, character: char },
{ code: "W100", line: this.line, character: originalChar },
checks,
function() { return true; }
);
Expand All @@ -1618,7 +1637,7 @@ Lexer.prototype = {
if (shouldTriggerError) {
this.triggerAsync(
"warning",
{ code: "W101", line: this.line, character: this.input.length },
{ code: "W101", line: this.line, character: this.originalInput.length },
checks,
function() { return true; }
);
Expand Down Expand Up @@ -1723,7 +1742,7 @@ Lexer.prototype = {
obj.type = obj.type || type;
obj.value = value;
obj.line = this.line;
obj.character = this.char;
obj.character = this.originalChar;
obj.from = this.from;
if (obj.identifier && token) obj.raw_text = token.text || token.value;
if (token && token.startLine && token.startLine !== this.line) {
Expand Down Expand Up @@ -1773,7 +1792,7 @@ Lexer.prototype = {
this.trigger("error", {
code: "E024",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ this.peek() ]
});

Expand Down Expand Up @@ -1862,22 +1881,22 @@ Lexer.prototype = {
this.trigger("warning", {
code: "W045",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ token.value ]
});
}

this.triggerAsync("warning", {
code: "W114",
line: this.line,
character: this.char,
character: this.originalChar,
data: [ "0x-" ]
}, checks, function() { return token.base === 16 && state.jsonMode; });

this.triggerAsync("warning", {
code: "W115",
line: this.line,
character: this.char
character: this.originalChar
}, checks, function() {
return state.isStrict() && token.base === 8 && token.isLegacy;
});
Expand Down Expand Up @@ -1905,7 +1924,7 @@ Lexer.prototype = {
type: token.commentType,
isSpecial: token.isSpecial,
line: this.line,
character: this.char,
character: this.originalChar,
from: this.from
};
}
Expand Down

0 comments on commit 07513d5

Please sign in to comment.