From 2062747282afda73f92d41bd870db8ee1eaa541c Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 7 Feb 2020 14:39:26 -0800 Subject: [PATCH 01/10] feat: add options.skipEncoding() to pug-load.read() This allows Pug to correctly read non-utf8 files, if the user wants to do such a thing. The optional function takes the same arguments as load.read(), and should return a truthy/falsey value. If truthy, load.read() will NOT specify an encoding when it reads the file from disk. Otherwise, it will continue to specify "utf-8" as the encoding. --- packages/pug-load/index.js | 7 ++++++- packages/pug/lib/index.js | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/pug-load/index.js b/packages/pug-load/index.js index 1eb1ffcb1..2335d47b9 100644 --- a/packages/pug-load/index.js +++ b/packages/pug-load/index.js @@ -80,7 +80,12 @@ load.resolve = function resolve(filename, source, options) { return filename; }; load.read = function read(filename, options) { - return fs.readFileSync(filename, 'utf8'); + if (options && options.skipEncoding && options.skipEncoding(filename, options)) { + return fs.readFileSync(filename); + } + else { + return fs.readFileSync(filename, 'utf8'); + } }; load.validateOptions = function validateOptions(options) { diff --git a/packages/pug/lib/index.js b/packages/pug/lib/index.js index ec734f7bb..b20e88f9b 100644 --- a/packages/pug/lib/index.js +++ b/packages/pug/lib/index.js @@ -82,6 +82,7 @@ function compileBody(str, options) { var ast = load.string(str, { filename: options.filename, basedir: options.basedir, + skipEncoding: options.skipEncoding, lex: function(str, options) { var lexOptions = {}; Object.keys(options).forEach(function(key) { @@ -268,6 +269,7 @@ exports.compile = function(str, options) { compileDebug: options.compileDebug !== false, filename: options.filename, basedir: options.basedir, + skipEncoding: options.skipEncoding, pretty: options.pretty, doctype: options.doctype, inlineRuntimeFunctions: options.inlineRuntimeFunctions, From df0a1e120218eff4b3b73f1e4dfe2c0fe87e1c81 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 5 May 2020 20:19:18 -0700 Subject: [PATCH 02/10] Move encoding of file from load.read() to load.file() --- packages/pug-load/index.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/pug-load/index.js b/packages/pug-load/index.js index 2335d47b9..246e6f68f 100644 --- a/packages/pug-load/index.js +++ b/packages/pug-load/index.js @@ -56,7 +56,7 @@ load.file = function loadFile(filename, options) { options = assign(getOptions(options), { filename: filename, }); - var str = options.read(filename); + var str = options.read(filename).toString('utf8'); return load.string(str, options); }; @@ -80,12 +80,7 @@ load.resolve = function resolve(filename, source, options) { return filename; }; load.read = function read(filename, options) { - if (options && options.skipEncoding && options.skipEncoding(filename, options)) { - return fs.readFileSync(filename); - } - else { - return fs.readFileSync(filename, 'utf8'); - } + return fs.readFileSync(filename); }; load.validateOptions = function validateOptions(options) { From f8bd0023cb63f2c0285709bceda51f967e4f4ff5 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 5 May 2020 20:20:29 -0700 Subject: [PATCH 03/10] Keep a copy of each loaded file as the raw Buffer This lives next to the existing `file.str` property. --- packages/pug-load/index.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/pug-load/index.js b/packages/pug-load/index.js index 246e6f68f..92ae0395c 100644 --- a/packages/pug-load/index.js +++ b/packages/pug-load/index.js @@ -21,16 +21,18 @@ function load(ast, options) { if (file.type !== 'FileReference') { throw new Error('Expected file.type to be "FileReference"'); } - var path, str; + var path, str, raw; try { path = options.resolve(file.path, file.filename, options); file.fullPath = path; - str = options.read(path, options); + raw = options.read(path, options); + str = raw.toString('utf8'); } catch (ex) { ex.message += '\n at ' + node.filename + ' line ' + node.line; throw ex; } file.str = str; + file.raw = raw; if (node.type === 'Extends' || node.type === 'Include') { file.ast = load.string( str, From 8ea501e3b3af69e14549efe43f9d24c5373be245 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 5 May 2020 20:27:07 -0700 Subject: [PATCH 04/10] Update jest snapshot for pug-load --- .../test/__snapshots__/index.test.js.snap | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/packages/pug-load/test/__snapshots__/index.test.js.snap b/packages/pug-load/test/__snapshots__/index.test.js.snap index 1fd0b4beb..833ab940e 100644 --- a/packages/pug-load/test/__snapshots__/index.test.js.snap +++ b/packages/pug-load/test/__snapshots__/index.test.js.snap @@ -29,6 +29,22 @@ Object { "fullPath": "/bar.pug", "line": 1, "path": "bar.pug", + "raw": Object { + "data": Array [ + 98, + 108, + 111, + 99, + 107, + 32, + 98, + 105, + 110, + 103, + 10, + ], + "type": "Buffer", + }, "str": "block bing ", "type": "FileReference", @@ -99,6 +115,22 @@ Object { "fullPath": "/bing.pug", "line": 4, "path": "bing.pug", + "raw": Object { + "data": Array [ + 46, + 98, + 105, + 110, + 103, + 32, + 98, + 111, + 110, + 103, + 10, + ], + "type": "Buffer", + }, "str": ".bing bong ", "type": "FileReference", @@ -122,6 +154,43 @@ Object { "fullPath": "/script.js", "line": 6, "path": "script.js", + "raw": Object { + "data": Array [ + 100, + 111, + 99, + 117, + 109, + 101, + 110, + 116, + 46, + 119, + 114, + 105, + 116, + 101, + 40, + 39, + 104, + 101, + 108, + 108, + 111, + 32, + 119, + 111, + 114, + 108, + 100, + 33, + 39, + 41, + 59, + 10, + ], + "type": "Buffer", + }, "str": "document.write('hello world!'); ", "type": "FileReference", From 6d990129c2add974aa4fb36c71802db1717e4e4c Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 5 May 2020 20:20:24 -0700 Subject: [PATCH 05/10] Check for `renderBuffer` property of filter to pass Buffer to the filter --- packages/pug-filters/lib/handle-filters.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/pug-filters/lib/handle-filters.js b/packages/pug-filters/lib/handle-filters.js index 6696290e8..b0baeddad 100644 --- a/packages/pug-filters/lib/handle-filters.js +++ b/packages/pug-filters/lib/handle-filters.js @@ -24,9 +24,8 @@ function handleFilters(ast, filters, options, filterAliases) { var firstFilter = node.filters.pop(); var attrs = getAttributes(firstFilter, options); var filename = (attrs.filename = node.file.fullPath); - var str = node.file.str; node.type = 'Text'; - node.val = filterFileWithFallback(firstFilter, filename, str, attrs); + node.val = filterFileWithFallback(firstFilter, filename, node.file, attrs); node.filters .slice() .reverse() @@ -55,10 +54,15 @@ function handleFilters(ast, filters, options, filterAliases) { } } - function filterFileWithFallback(filter, filename, text, attrs) { + function filterFileWithFallback(filter, filename, file, attrs) { var filterName = getFilterName(filter); if (filters && filters[filterName]) { - return filters[filterName](text, attrs); + if (filters[filterName].renderBuffer) { + return filters[filterName].renderBuffer(file.content, attrs); + } + else { + return filters[filterName](file.str, attrs); + } } else { return filterWithFallback(filter, filename, attrs, 'renderFile'); } From 1db19d268e97ef2eb78ccc02197243e059624b88 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 5 May 2020 20:34:46 -0700 Subject: [PATCH 06/10] Remove un-needed `skipEncoding` property from allowed options --- packages/pug/lib/index.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/pug/lib/index.js b/packages/pug/lib/index.js index b20e88f9b..ec734f7bb 100644 --- a/packages/pug/lib/index.js +++ b/packages/pug/lib/index.js @@ -82,7 +82,6 @@ function compileBody(str, options) { var ast = load.string(str, { filename: options.filename, basedir: options.basedir, - skipEncoding: options.skipEncoding, lex: function(str, options) { var lexOptions = {}; Object.keys(options).forEach(function(key) { @@ -269,7 +268,6 @@ exports.compile = function(str, options) { compileDebug: options.compileDebug !== false, filename: options.filename, basedir: options.basedir, - skipEncoding: options.skipEncoding, pretty: options.pretty, doctype: options.doctype, inlineRuntimeFunctions: options.inlineRuntimeFunctions, From c303124b555f793eb6ec2d9e5af06d09d77a63eb Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 5 May 2020 20:35:41 -0700 Subject: [PATCH 07/10] Fix wrong var name --- packages/pug-filters/lib/handle-filters.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pug-filters/lib/handle-filters.js b/packages/pug-filters/lib/handle-filters.js index b0baeddad..7b0bd9f26 100644 --- a/packages/pug-filters/lib/handle-filters.js +++ b/packages/pug-filters/lib/handle-filters.js @@ -58,7 +58,7 @@ function handleFilters(ast, filters, options, filterAliases) { var filterName = getFilterName(filter); if (filters && filters[filterName]) { if (filters[filterName].renderBuffer) { - return filters[filterName].renderBuffer(file.content, attrs); + return filters[filterName].renderBuffer(file.raw, attrs); } else { return filters[filterName](file.str, attrs); From 34e172ec5d7ea618ac8263f680cdd88ad01922ae Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 5 May 2020 20:40:17 -0700 Subject: [PATCH 08/10] Format to make prettier feel pretty --- packages/pug-filters/lib/handle-filters.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/pug-filters/lib/handle-filters.js b/packages/pug-filters/lib/handle-filters.js index 7b0bd9f26..884a50138 100644 --- a/packages/pug-filters/lib/handle-filters.js +++ b/packages/pug-filters/lib/handle-filters.js @@ -25,7 +25,12 @@ function handleFilters(ast, filters, options, filterAliases) { var attrs = getAttributes(firstFilter, options); var filename = (attrs.filename = node.file.fullPath); node.type = 'Text'; - node.val = filterFileWithFallback(firstFilter, filename, node.file, attrs); + node.val = filterFileWithFallback( + firstFilter, + filename, + node.file, + attrs + ); node.filters .slice() .reverse() @@ -59,8 +64,7 @@ function handleFilters(ast, filters, options, filterAliases) { if (filters && filters[filterName]) { if (filters[filterName].renderBuffer) { return filters[filterName].renderBuffer(file.raw, attrs); - } - else { + } else { return filters[filterName](file.str, attrs); } } else { From 425f44a7d443748de4308c0c3c0a50cf973f427e Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 6 May 2020 09:06:54 -0700 Subject: [PATCH 09/10] Serialize "Buffer" types for better readability in jest snapshots These render as objects with `type: "Buffer"`, along with the `size` and `hash` of the data in the buffer. --- package.json | 3 +- .../test/__snapshots__/index.test.js.snap | 66 ++----------------- scripts/buffer-serializer.js | 19 ++++++ 3 files changed, 27 insertions(+), 61 deletions(-) create mode 100644 scripts/buffer-serializer.js diff --git a/package.json b/package.json index 1087ae2d9..5988a3c4f 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,8 @@ "testEnvironment": "node", "snapshotSerializers": [ "./scripts/filename-serializer.js", - "./scripts/prettier-javascript-serializer.js" + "./scripts/prettier-javascript-serializer.js", + "./scripts/buffer-serializer.js" ] }, "license": "MIT", diff --git a/packages/pug-load/test/__snapshots__/index.test.js.snap b/packages/pug-load/test/__snapshots__/index.test.js.snap index 833ab940e..6ad59e8c7 100644 --- a/packages/pug-load/test/__snapshots__/index.test.js.snap +++ b/packages/pug-load/test/__snapshots__/index.test.js.snap @@ -30,19 +30,8 @@ Object { "line": 1, "path": "bar.pug", "raw": Object { - "data": Array [ - 98, - 108, - 111, - 99, - 107, - 32, - 98, - 105, - 110, - 103, - 10, - ], + "hash": "538bf7d4b81ef364b1f2e9d42c11f156", + "size": 11, "type": "Buffer", }, "str": "block bing @@ -116,19 +105,8 @@ Object { "line": 4, "path": "bing.pug", "raw": Object { - "data": Array [ - 46, - 98, - 105, - 110, - 103, - 32, - 98, - 111, - 110, - 103, - 10, - ], + "hash": "58ecbe086e7a045084cbddac849a2563", + "size": 11, "type": "Buffer", }, "str": ".bing bong @@ -155,40 +133,8 @@ Object { "line": 6, "path": "script.js", "raw": Object { - "data": Array [ - 100, - 111, - 99, - 117, - 109, - 101, - 110, - 116, - 46, - 119, - 114, - 105, - 116, - 101, - 40, - 39, - 104, - 101, - 108, - 108, - 111, - 32, - 119, - 111, - 114, - 108, - 100, - 33, - 39, - 41, - 59, - 10, - ], + "hash": "86d4f8e34165faeb09f10255121078f8", + "size": 32, "type": "Buffer", }, "str": "document.write('hello world!'); diff --git a/scripts/buffer-serializer.js b/scripts/buffer-serializer.js new file mode 100644 index 000000000..f74ade0f0 --- /dev/null +++ b/scripts/buffer-serializer.js @@ -0,0 +1,19 @@ +const crypto = require('crypto'); + +// Buffer serializer to reduce snapshot gore for Node Buffer type +module.exports = { + test: function(val) { + return ( + val && + Buffer.isBuffer(val) + ); + }, + print: function(val, serialize, indent) { + const output = { + type: 'Buffer', + size: val.length, + hash: crypto.createHash('md5').update(val).digest('hex') + }; + return serialize(output); + }, +}; From 017711cfe2701bd229883c5275de9e20e7908486 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 6 May 2020 09:09:54 -0700 Subject: [PATCH 10/10] Appease prettier --- scripts/buffer-serializer.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/buffer-serializer.js b/scripts/buffer-serializer.js index f74ade0f0..ace36985d 100644 --- a/scripts/buffer-serializer.js +++ b/scripts/buffer-serializer.js @@ -3,16 +3,16 @@ const crypto = require('crypto'); // Buffer serializer to reduce snapshot gore for Node Buffer type module.exports = { test: function(val) { - return ( - val && - Buffer.isBuffer(val) - ); + return val && Buffer.isBuffer(val); }, print: function(val, serialize, indent) { const output = { type: 'Buffer', size: val.length, - hash: crypto.createHash('md5').update(val).digest('hex') + hash: crypto + .createHash('md5') + .update(val) + .digest('hex'), }; return serialize(output); },