Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rewrite pug-code-gen as an babel AST generator #3019

Open
wants to merge 24 commits into
base: master
Choose a base branch
from
Open

Rewrite pug-code-gen as an babel AST generator #3019

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
2ff672e
Rewrite pug-code-gen as an babel AST generator
jeromew Jun 1, 2018
486a3d8
remove emptyStatement in return
jeromew Jun 4, 2018
5cf03d8
Remove buffer optimization (should be move to a babel plugin)
jeromew Jun 5, 2018
d66116d
Refactor to make visitors return an AST array
jeromew Jun 6, 2018
ab49478
replace concat with push.apply
jeromew Jun 7, 2018
4dba99b
Add babel plugin for buffer assignment compaction
jeromew Jun 7, 2018
d2089a1
babel plugin: use mode strict
jeromew Jun 7, 2018
81a98c9
lexer can now pass AST to code-gen (avoid double parsing)
jeromew Jun 7, 2018
00fb754
Improve ast lexer --> code-gen path
jeromew Jun 8, 2018
6dc8edd
remove mention of acorn
jeromew Jun 8, 2018
b2e996d
use preprocessed AST in mixin args
jeromew Jun 8, 2018
76e51b7
perf: modify babel-plugin-transform-with usage
jeromew Jun 18, 2018
f5267a0
remove babel transformFromAst (perf). Custom code for with and compac…
jeromew Oct 4, 2018
cca9a97
Upgrade with to 6.0.0
jeromew Oct 4, 2018
43e756b
Remove babel plugins. Remove useless code optimization.
jeromew Feb 1, 2019
11965fc
Extract custom `with` code in a method
jeromew Feb 2, 2019
cb8b447
rebase babel branch on master
jeromew Jun 13, 2020
8c99d16
format code
jeromew Jun 13, 2020
f9f235e
Upgrade to babel 7
jeromew Jun 13, 2020
55ddab1
Improve performance
jeromew Jun 15, 2020
c782102
Fix format
jeromew Jun 15, 2020
0202c28
Improve performance
jeromew Jun 16, 2020
3f26f90
Replace stringify with @babel/generator's jsescOption isScriptContext
jeromew Jun 16, 2020
3d729e1
Improve postprocess speed by avoiding compacted branches
jeromew Jun 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1,238 changes: 906 additions & 332 deletions packages/pug-code-gen/index.js
View file
Open in desktop

Large diffs are not rendered by default.

7 changes: 6 additions & 1 deletion packages/pug-code-gen/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,19 @@
"pug"
],
"dependencies": {
"@babel/core": "^7.10.2",
"@babel/generator": "^7.10.2",
"@babel/template": "^7.10.1",
"@babel/types": "^7.10.2",
"@babel/parser": "^7.10.2",
"constantinople": "^4.0.1",
"doctypes": "^1.1.0",
"js-stringify": "^1.0.2",
"pug-attrs": "^2.0.4",
"pug-error": "^1.3.3",
"pug-runtime": "^2.0.5",
"void-elements": "^3.1.0",
"with": "^7.0.0"
"with": "^7.0.2"
},
"files": [
"index.js"
Expand Down
59 changes: 32 additions & 27 deletions packages/pug-lexer/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
'use strict';

var assert = require('assert');
var isExpression = require('is-expression');
var babylon = require('@babel/parser');
var characterParser = require('character-parser');
var error = require('pug-error');

Expand Down Expand Up @@ -72,21 +72,17 @@ Lexer.prototype = {
if (!value) this.error('ASSERT_FAILED', message);
},

isExpression: function(exp) {
return isExpression(exp, {
throw: true,
});
parseExpression: function(exp) {
return babylon.parseExpression(exp);
},

assertExpression: function(exp, noThrow) {
//this verifies that a JavaScript expression is valid
try {
this.callLexerFunction('isExpression', exp);
return true;
return this.callLexerFunction('parseExpression', exp);
} catch (ex) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We may need to update the error handling code. Specifically the comment saying:

not coming from acorn

is definitely out of date now.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i removed the message. The rest of the code is working.

if (noThrow) return false;

// not coming from acorn
if (!ex.loc) throw ex;

this.incrementLine(ex.loc.line - 1);
Expand Down Expand Up @@ -118,7 +114,7 @@ Lexer.prototype = {
* @api private
*/

tok: function(type, val) {
tok: function(type, val, ast) {
var res = {
type: type,
loc: {
Expand All @@ -131,6 +127,7 @@ Lexer.prototype = {
};

if (val !== undefined) res.val = val;
if (ast !== undefined) res.ast = ast;

return res;
},
Expand Down Expand Up @@ -367,13 +364,14 @@ Lexer.prototype = {
*/

interpolation: function() {
var ast;
if (/^#\{/.test(this.input)) {
var match = this.bracketExpression(1);
this.consume(match.end + 1);
var tok = this.tok('interpolation', match.src);
ast = this.assertExpression(match.src);
var tok = this.tok('interpolation', match.src, ast);
this.tokens.push(tok);
this.incrementColumn(2); // '#{'
this.assertExpression(match.src);

var splitted = match.src.split('\n');
var lines = splitted.length - 1;
Expand Down Expand Up @@ -619,7 +617,7 @@ Lexer.prototype = {
tok.mustEscape = matchOfStringInterp[2] === '#';
tok.buffer = true;
tok.val = range.src;
this.assertExpression(range.src);
tok.ast = this.assertExpression(range.src);

if (range.end + 1 < rest.length) {
rest = rest.substr(range.end + 1);
Expand Down Expand Up @@ -851,7 +849,7 @@ Lexer.prototype = {
var tok = this.scanEndOfLine(/^case +([^\n]+)/, 'case');
if (tok) {
this.incrementColumn(-tok.val.length);
this.assertExpression(tok.val);
tok.ast = this.assertExpression(tok.val);
this.incrementColumn(tok.val.length);
this.tokens.push(this.tokEnd(tok));
return true;
Expand Down Expand Up @@ -880,7 +878,7 @@ Lexer.prototype = {
}

this.incrementColumn(-tok.val.length);
this.assertExpression(tok.val);
tok.ast = this.assertExpression(tok.val);
this.incrementColumn(tok.val.length);
this.tokens.push(this.tokEnd(tok));
return true;
Expand Down Expand Up @@ -913,7 +911,7 @@ Lexer.prototype = {
*/

call: function() {
var tok, captures, increment;
var tok, ast, captures, increment;
if ((captures = /^\+(\s*)(([-\w]+)|(#\{))/.exec(this.input))) {
// try to consume simple or interpolated call
if (captures[3]) {
Expand All @@ -926,8 +924,8 @@ Lexer.prototype = {
var match = this.bracketExpression(2 + captures[1].length);
increment = match.end + 1;
this.consume(increment);
this.assertExpression(match.src);
tok = this.tok('call', '#{' + match.src + '}');
ast = this.assertExpression(match.src);
tok = this.tok('call', '#{' + match.src + '}', ast);
}

this.incrementColumn(increment);
Expand All @@ -941,7 +939,7 @@ Lexer.prototype = {
this.incrementColumn(1);
this.consume(range.end + 1);
tok.args = range.src;
this.assertExpression('[' + tok.args + ']');
tok.ast_args = this.assertExpression('[' + tok.args + ']').elements;
for (var i = 0; i <= tok.args.length; i++) {
if (tok.args[i] === '\n') {
this.incrementLine(1);
Expand Down Expand Up @@ -989,11 +987,11 @@ Lexer.prototype = {
switch (type) {
case 'if':
case 'else-if':
this.assertExpression(js);
tok.ast = this.assertExpression(js);
break;
case 'unless':
this.assertExpression(js);
tok.val = '!(' + js + ')';
tok.ast = this.assertExpression(tok.val);
tok.type = 'if';
break;
case 'else':
Expand All @@ -1016,11 +1014,11 @@ Lexer.prototype = {
*/

while: function() {
var captures, tok;
var captures, ast, tok;
if ((captures = /^while +([^\n]+)/.exec(this.input))) {
this.consume(captures[0].length);
this.assertExpression(captures[1]);
tok = this.tok('while', captures[1]);
ast = this.assertExpression(captures[1]);
tok = this.tok('while', captures[1], ast);
this.incrementColumn(captures[0].length);
this.tokens.push(this.tokEnd(tok));
return true;
Expand All @@ -1045,7 +1043,7 @@ Lexer.prototype = {
var tok = this.tok('each', captures[1]);
tok.key = captures[2] || null;
this.incrementColumn(captures[0].length - captures[3].length);
this.assertExpression(captures[3]);
tok.ast = this.assertExpression(captures[3]);
tok.code = captures[3];
this.incrementColumn(captures[3].length);
this.tokens.push(this.tokEnd(tok));
Expand Down Expand Up @@ -1172,7 +1170,7 @@ Lexer.prototype = {
// ---- captures[0] - captures[2]
// ^ after colno
this.incrementColumn(captures[0].length - captures[2].length);
if (tok.buffer) this.assertExpression(code);
if (tok.buffer) tok.ast = this.assertExpression(code);
this.tokens.push(tok);

// p #[!= abc] hey
Expand Down Expand Up @@ -1271,6 +1269,7 @@ Lexer.prototype = {

if (valueResponse.val) {
tok.val = valueResponse.val;
tok.ast = valueResponse.ast;
tok.mustEscape = valueResponse.mustEscape;
} else {
// was a boolean attribute (ex: `input(disabled)`)
Expand Down Expand Up @@ -1312,6 +1311,7 @@ Lexer.prototype = {
var state = characterParser.defaultState();
var col = this.colno;
var line = this.lineno;
var ast;

// consume all whitespace before the equals sign
for (i = 0; i < str.length; i++) {
Expand Down Expand Up @@ -1420,12 +1420,17 @@ Lexer.prototype = {
}
}

this.assertExpression(val);
ast = this.assertExpression(val);

this.lineno = line;
this.colno = col;

return {val: val, mustEscape: escapeAttr, remainingSource: str.substr(i)};
return {
val: val,
ast: ast,
mustEscape: escapeAttr,
remainingSource: str.substr(i),
};
},

/**
Expand Down
4 changes: 1 addition & 3 deletions packages/pug-lexer/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,11 @@
"pug"
],
"dependencies": {
"@babel/parser": "^7.10.2",
"character-parser": "^2.2.0",
"is-expression": "^4.0.0",
"pug-error": "^1.3.3"
},
"devDependencies": {
"acorn": "^7.1.1",
"acorn-walk": "^7.1.1"
},
"files": [
"index.js",
Expand Down