diff --git a/.eslintrc.js b/.eslintrc.js index b73d0a97db..930b631331 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -2,7 +2,8 @@ module.exports = { "env": { "browser": true, "es6": true, - "node": true + "node": true, + "mocha": true }, "extends": [ "eslint:recommended", diff --git a/CHANGES.md b/CHANGES.md index d8cbfee4b4..e1aa893c8f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ Parser Engine: +- (parser) Adds SHEBANG utility mode [Josh Goebel][] - (enh) Added `on:begin` callback for modes (#2261) [Josh Goebel][] - (enh) Added `on:end` callback for modes (#2261) [Josh Goebel][] - (enh) Added ability to programatically ignore begin and end matches (#2261) [Josh Goebel][] @@ -13,6 +14,7 @@ Deprecations: Language Improvements: +- enh(typescript/javascript/coffeescript/livescript) derive ECMAscript keywords from a common foudation (#2518) [Josh Goebel][] - enh(typescript) add setInterval, setTimeout, clearInterval, clearTimeout (#2514) [Josh Goebel][] - enh(javascript) add setInterval, setTimeout, clearInterval, clearTimeout (#2514) [Vania Kucher][] - fix(javascript) prevent `set` keyword conflicting with setTimeout, etc. (#2514) [Vania Kucher][] diff --git a/src/languages/bash.js b/src/languages/bash.js index be9c8c92fc..e96a46dfd8 100644 --- a/src/languages/bash.js +++ b/src/languages/bash.js @@ -55,11 +55,21 @@ export default function(hljs) { VAR ] }; - const SHEBANG = { - className: 'meta', - begin: /^#![^\n]+sh\s*$/, + const SH_LIKE_SHELLS = [ + "fish", + "bash", + "zsh", + "sh", + "csh", + "ksh", + "tcsh", + "dash", + "scsh", + ]; + const KNOWN_SHEBANG = hljs.SHEBANG({ + binary: `(${SH_LIKE_SHELLS.join("|")})`, relevance: 10 - }; + }); const FUNCTION = { className: 'function', begin: /\w[\w\d_]*\s*\(\s*\)\s*\{/, @@ -98,7 +108,8 @@ export default function(hljs) { '-ne -eq -lt -gt -f -d -e -s -l -a' // relevance booster }, contains: [ - SHEBANG, + KNOWN_SHEBANG, // to catch known shells and boost relevancy + hljs.SHEBANG(), // to catch unknown shells but still highlight the shebang FUNCTION, ARITHMETIC, hljs.HASH_COMMENT_MODE, diff --git a/src/languages/coffeescript.js b/src/languages/coffeescript.js index 0defcd5adb..60c8bb5a4a 100644 --- a/src/languages/coffeescript.js +++ b/src/languages/coffeescript.js @@ -7,21 +7,45 @@ Category: common, scripting Website: https://coffeescript.org */ +import * as ECMAScript from "./lib/ecmascript"; + export default function(hljs) { + var COFFEE_BUILT_INS = [ + 'npm', + 'print' + ]; + var COFFEE_LITERALS = [ + 'yes', + 'no', + 'on', + 'off' + ]; + var COFFEE_KEYWORDS = [ + 'then', + 'unless', + 'until', + 'loop', + 'by', + 'when', + 'and', + 'or', + 'is', + 'isnt', + 'not' + ]; + var NOT_VALID_KEYWORDS = [ + "var", + "const", + "let", + "function", + "static" + ]; + var excluding = (list) => + (kw) => !list.includes(kw); var KEYWORDS = { - keyword: - // JS keywords - 'in if for while finally new do return else break catch instanceof throw try this ' + - 'switch continue typeof delete debugger super yield import export from as default await ' + - // Coffee keywords - 'then unless until loop of by when and or is isnt not', - literal: - // JS literals - 'true false null undefined ' + - // Coffee literals - 'yes no on off', - built_in: - 'npm require console print module global window document' + keyword: ECMAScript.KEYWORDS.concat(COFFEE_KEYWORDS).filter(excluding(NOT_VALID_KEYWORDS)).join(" "), + literal: ECMAScript.LITERALS.concat(COFFEE_LITERALS).join(" "), + built_in: ECMAScript.BUILT_INS.concat(COFFEE_BUILT_INS).join(" ") }; var JS_IDENT_RE = '[A-Za-z$_][0-9A-Za-z$_]*'; var SUBST = { diff --git a/src/languages/hy.js b/src/languages/hy.js index 4faa8780f8..027502a6c6 100644 --- a/src/languages/hy.js +++ b/src/languages/hy.js @@ -45,11 +45,6 @@ export default function(hljs) { var SYMBOL_RE = '[' + SYMBOLSTART + '][' + SYMBOLSTART + '0-9/;:]*'; var SIMPLE_NUMBER_RE = '[-+]?\\d+(\\.\\d+)?'; - var SHEBANG = { - className: 'meta', - begin: '^#!', end: '$' - }; - var SYMBOL = { begin: SYMBOL_RE, relevance: 0 @@ -105,6 +100,6 @@ export default function(hljs) { name: 'Hy', aliases: ['hylang'], illegal: /\S/, - contains: [SHEBANG, LIST, STRING, HINT, HINT_COL, COMMENT, KEY, COLLECTION, NUMBER, LITERAL] + contains: [hljs.SHEBANG(), LIST, STRING, HINT, HINT_COL, COMMENT, KEY, COLLECTION, NUMBER, LITERAL] } } diff --git a/src/languages/javascript.js b/src/languages/javascript.js index eb0d62adc0..9f900569c2 100644 --- a/src/languages/javascript.js +++ b/src/languages/javascript.js @@ -5,6 +5,8 @@ Category: common, scripting Website: https://developer.mozilla.org/en-US/docs/Web/JavaScript */ +import * as ECMAScript from "./lib/ecmascript"; + export default function(hljs) { var FRAGMENT = { begin: '<>', @@ -16,24 +18,9 @@ export default function(hljs) { }; var IDENT_RE = '[A-Za-z$_][0-9A-Za-z$_]*'; var KEYWORDS = { - keyword: - 'in of if for while finally var new function do return void else break catch ' + - 'instanceof with throw case default try this switch continue typeof delete ' + - 'let yield const export super debugger as async await static ' + - // ECMAScript 6 modules import - 'import from as' - , - literal: - 'true false null undefined NaN Infinity', - built_in: - 'eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent ' + - 'encodeURI encodeURIComponent escape unescape Object Function Boolean Error ' + - 'EvalError InternalError RangeError ReferenceError StopIteration SyntaxError ' + - 'TypeError URIError Number Math Date String RegExp Array Float32Array ' + - 'Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array ' + - 'Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require ' + - 'module console window document Symbol Set Map WeakSet WeakMap Proxy Reflect ' + - 'Promise setInterval setTimeout clearInterval clearTimeout' + keyword: ECMAScript.KEYWORDS.join(" "), + literal: ECMAScript.LITERALS.join(" "), + built_in: ECMAScript.BUILT_INS.join(" ") }; var NUMBER = { className: 'number', @@ -110,15 +97,15 @@ export default function(hljs) { aliases: ['js', 'jsx', 'mjs', 'cjs'], keywords: KEYWORDS, contains: [ + hljs.SHEBANG({ + binary: "node", + relevance: 5 + }), { className: 'meta', relevance: 10, begin: /^\s*['"]use (strict|asm)['"]/ }, - { - className: 'meta', - begin: /^#!/, end: /$/ - }, hljs.APOS_STRING_MODE, hljs.QUOTE_STRING_MODE, HTML_TEMPLATE, diff --git a/src/languages/lib/ecmascript.js b/src/languages/lib/ecmascript.js new file mode 100644 index 0000000000..0e80f2c40c --- /dev/null +++ b/src/languages/lib/ecmascript.js @@ -0,0 +1,138 @@ +const KEYWORDS = [ + "as", // for exports + "in", + "of", + "if", + "for", + "while", + "finally", + "var", + "new", + "function", + "do", + "return", + "void", + "else", + "break", + "catch", + "instanceof", + "with", + "throw", + "case", + "default", + "try", + "switch", + "continue", + "typeof", + "delete", + "let", + "yield", + "const", + "class", + // JS handles these with a special rule + // "get", + // "set", + "debugger", + "async", + "await", + "static", + "import", + "from", + "export", + "extends" +]; +const LITERALS = [ + "true", + "false", + "null", + "undefined", + "NaN", + "Infinity" +]; + +const TYPES = [ + "Intl", + "DataView", + "Number", + "Math", + "Date", + "String", + "RegExp", + "Object", + "Function", + "Boolean", + "Error", + "Symbol", + "Set", + "Map", + "WeakSet", + "WeakMap", + "Proxy", + "Reflect", + "JSON", + "Promise", + "Float64Array", + "Int16Array", + "Int32Array", + "Int8Array", + "Uint16Array", + "Uint32Array", + "Float32Array", + "Array", + "Uint8Array", + "Uint8ClampedArray", + "ArrayBuffer" +]; + +const ERROR_TYPES = [ + "EvalError", + "InternalError", + "RangeError", + "ReferenceError", + "SyntaxError", + "TypeError", + "URIError" +]; + +const BUILT_IN_GLOBALS = [ + "setInterval", + "setTimeout", + "clearInterval", + "clearTimeout", + + "require", + "exports", + + "eval", + "isFinite", + "isNaN", + "parseFloat", + "parseInt", + "decodeURI", + "decodeURIComponent", + "encodeURI", + "encodeURIComponent", + "escape", + "unescape" +]; + +const BUILT_IN_VARIABLES = [ + "arguments", + "this", + "super", + "console", + "window", + "document", + "localStorage", + "module", + "global" // Node.js +]; + +const BUILT_INS = [].concat( + BUILT_IN_GLOBALS, + BUILT_IN_VARIABLES, + TYPES, + ERROR_TYPES +); + +export { LITERALS, BUILT_INS, KEYWORDS }; diff --git a/src/languages/lisp.js b/src/languages/lisp.js index c8305f6bea..5b505cd1a5 100644 --- a/src/languages/lisp.js +++ b/src/languages/lisp.js @@ -9,10 +9,6 @@ export default function(hljs) { var LISP_IDENT_RE = '[a-zA-Z_\\-\\+\\*\\/\\<\\=\\>\\&\\#][a-zA-Z0-9_\\-\\+\\*\\/\\<\\=\\>\\&\\#!]*'; var MEC_RE = '\\|[^]*?\\|'; var LISP_SIMPLE_NUMBER_RE = '(\\-|\\+)?\\d+(\\.\\d+|\\/\\d+)?((d|e|f|l|s|D|E|F|L|S)(\\+|\\-)?\\d+)?'; - var SHEBANG = { - className: 'meta', - begin: '^#!', end: '$' - }; var LITERAL = { className: 'literal', begin: '\\b(t{1}|nil)\\b' @@ -97,7 +93,7 @@ export default function(hljs) { illegal: /\S/, contains: [ NUMBER, - SHEBANG, + hljs.SHEBANG(), LITERAL, STRING, COMMENT, diff --git a/src/languages/livescript.js b/src/languages/livescript.js index 17e6ab057d..4f1f072a90 100644 --- a/src/languages/livescript.js +++ b/src/languages/livescript.js @@ -8,23 +8,57 @@ Website: https://livescript.net Category: scripting */ +import * as ECMAScript from "./lib/ecmascript"; + export default function(hljs) { + var LIVESCRIPT_BUILT_INS = [ + 'npm', + 'print' + ]; + var LIVESCRIPT_LITERALS = [ + 'yes', + 'no', + 'on', + 'off', + 'it', + 'that', + 'void' + ]; + var LIVESCRIPT_KEYWORDS = [ + 'then', + 'unless', + 'until', + 'loop', + 'of', + 'by', + 'when', + 'and', + 'or', + 'is', + 'isnt', + 'not', + 'it', + 'that', + 'otherwise', + 'from', + 'to', + 'til', + 'fallthrough', + 'case', + 'enum', + 'native', + 'list', + 'map', + '__hasProp', + '__extends', + '__slice', + '__bind', + '__indexOf' + ]; var KEYWORDS = { - keyword: - // JS keywords - 'in if for while finally new do return else break catch instanceof throw try this ' + - 'switch continue typeof delete debugger case default function var with ' + - // LiveScript keywords - 'then unless until loop of by when and or is isnt not it that otherwise from to til fallthrough super ' + - 'case default function var void const let enum export import native list map ' + - '__hasProp __extends __slice __bind __indexOf', - literal: - // JS literals - 'true false null undefined ' + - // LiveScript literals - 'yes no on off it that void', - built_in: - 'npm require console print module global window document' + keyword: ECMAScript.KEYWORDS.concat(LIVESCRIPT_KEYWORDS).join(" "), + literal: ECMAScript.LITERALS.concat(LIVESCRIPT_LITERALS).join(" "), + built_in: ECMAScript.BUILT_INS.concat(LIVESCRIPT_BUILT_INS).join(" ") }; var JS_IDENT_RE = '[A-Za-z$_](?:\-[0-9A-Za-z$_]|[0-9A-Za-z$_])*'; var TITLE = hljs.inherit(hljs.TITLE_MODE, {begin: JS_IDENT_RE}); diff --git a/src/languages/scheme.js b/src/languages/scheme.js index 335c8e14d0..df26b7cf73 100644 --- a/src/languages/scheme.js +++ b/src/languages/scheme.js @@ -50,12 +50,6 @@ export default function(hljs) { 'with-input-from-file with-output-to-file write write-char zero?' }; - var SHEBANG = { - className: 'meta', - begin: '^#!', - end: '$' - }; - var LITERAL = { className: 'literal', begin: '(#t|#f|#\\\\' + SCHEME_IDENT_RE + '|#\\\\.)' @@ -150,6 +144,6 @@ export default function(hljs) { return { name: 'Scheme', illegal: /\S/, - contains: [SHEBANG, NUMBER, STRING, QUOTED_IDENT, QUOTED_LIST, LIST].concat(COMMENT_MODES) + contains: [hljs.SHEBANG(), NUMBER, STRING, QUOTED_IDENT, QUOTED_LIST, LIST].concat(COMMENT_MODES) }; } diff --git a/src/languages/typescript.js b/src/languages/typescript.js index fd7f59ad7f..f830428477 100644 --- a/src/languages/typescript.js +++ b/src/languages/typescript.js @@ -7,26 +7,36 @@ Website: https://www.typescriptlang.org Category: common, scripting */ +import * as ECMAScript from "./lib/ecmascript"; + export default function(hljs) { var JS_IDENT_RE = '[A-Za-z$_][0-9A-Za-z$_]*'; + var TYPES = [ + "any", + "void", + "number", + "boolean", + "string", + "object", + "never", + "enum" + ]; + var TS_SPECIFIC_KEYWORDS = [ + "type", + "namespace", + "typedef", + "interface", + "public", + "private", + "protected", + "implements", + "declare", + "abstract" + ]; var KEYWORDS = { - keyword: - 'in if for while finally var new function do return void else break catch ' + - 'instanceof with throw case default try this switch continue typeof delete ' + - 'let yield const class public private protected get set super ' + - 'static implements enum export import declare type namespace abstract ' + - 'as from extends async await', - literal: - 'true false null undefined NaN Infinity', - built_in: - 'eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent ' + - 'encodeURI encodeURIComponent escape unescape Object Function Boolean Error ' + - 'EvalError InternalError RangeError ReferenceError StopIteration SyntaxError ' + - 'TypeError URIError Number Math Date String RegExp Array Float32Array ' + - 'Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array ' + - 'Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require ' + - 'module console window document any number boolean string void Promise ' + - 'setInterval setTimeout clearInterval clearTimeout' + keyword: ECMAScript.KEYWORDS.concat(TS_SPECIFIC_KEYWORDS).join(" "), + literal: ECMAScript.LITERALS.join(" "), + built_in: ECMAScript.BUILT_INS.concat(TYPES).join(" ") }; var DECORATOR = { className: 'meta', @@ -117,6 +127,7 @@ export default function(hljs) { aliases: ['ts'], keywords: KEYWORDS, contains: [ + hljs.SHEBANG(), { className: 'meta', begin: /^\s*['"]use strict['"]/ diff --git a/src/lib/modes.js b/src/lib/modes.js index 8acb8f0e51..602141e78a 100644 --- a/src/lib/modes.js +++ b/src/lib/modes.js @@ -1,4 +1,5 @@ import { inherit } from './utils'; +import * as regex from './regex'; // Common regexps export const IDENT_RE = '[a-zA-Z]\\w*'; @@ -8,6 +9,26 @@ export const C_NUMBER_RE = '(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+ export const BINARY_NUMBER_RE = '\\b(0b[01]+)'; // 0b... export const RE_STARTERS_RE = '!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~'; +export const SHEBANG = (opts = {}) => { + const beginShebang = /^#![ ]*\//; + if (opts.binary) { + opts.begin = regex.concat( + beginShebang, + /.*\b/, + opts.binary, + /\b.*/); + } + return inherit({ + className: 'meta', + begin: beginShebang, + end: /$/, + relevance: 0, + "on:begin": (m, resp) => { + if (m.index !== 0) resp.ignoreMatch(); + } + }, opts); +}; + // Common modes export const BACKSLASH_ESCAPE = { begin: '\\\\[\\s\\S]', relevance: 0 diff --git a/src/lib/regex.js b/src/lib/regex.js index c349485eaf..003dc9cbca 100644 --- a/src/lib/regex.js +++ b/src/lib/regex.js @@ -8,6 +8,11 @@ export function source(re) { return (re && re.source) || re; } +export function concat(...args) { + const joined = args.map((x) => source(x)).join(""); + return joined; +} + export function countMatchGroups(re) { return (new RegExp(re.toString() + '|')).exec('').length - 1; } diff --git a/test/detect/index.js b/test/detect/index.js index fec7bb3276..b6daf0f31e 100644 --- a/test/detect/index.js +++ b/test/detect/index.js @@ -1,46 +1,44 @@ 'use strict'; -delete require.cache[require.resolve('../../build')] -delete require.cache[require.resolve('../../build/lib/core')] +delete require.cache[require.resolve('../../build')]; +delete require.cache[require.resolve('../../build/lib/core')]; -const fs = require('fs').promises; -const hljs = require('../../build'); +const fs = require('fs').promises; +const hljs = require('../../build'); hljs.debugMode(); // tests run in debug mode so errors are raised -const path = require('path'); -const utility = require('../utility'); -const { getThirdPartyPackages } = require('../../tools/lib/external_language') +const path = require('path'); +const utility = require('../utility'); +const { getThirdPartyPackages } = require('../../tools/lib/external_language'); -function testAutoDetection(language, {detectPath}) { +function testAutoDetection(language, { detectPath }) { const languagePath = detectPath || utility.buildPath('detect', language); - it(`should be detected as ${language}`, async () => { + it(`should be detected as ${language}`, async() => { const dir = await fs.stat(languagePath); - dir.isDirectory().should.be.true; + dir.isDirectory().should.be.true(); - const filenames = await fs.readdir(languagePath) - const filesContent = await Promise.all(filenames - .map(function(example) { + const filenames = await fs.readdir(languagePath); + await Promise.all(filenames + .map(async function(example) { const filename = path.join(languagePath, example); - return fs.readFile(filename, 'utf-8'); - })) - filesContent.forEach(function(content) { - const expected = language, - actual = hljs.highlightAuto(content).language; + const content = await fs.readFile(filename, 'utf-8'); + const detectedLanguage = hljs.highlightAuto(content).language; - actual.should.equal(expected); - }); + detectedLanguage.should.equal(language, + `${path.basename(filename)} should be detected as ${language}, but was ${detectedLanguage}`); + })); }); } describe('hljs.highlightAuto()', () => { - before( async function() { - let thirdPartyPackages = await getThirdPartyPackages(); + before(async function() { + const thirdPartyPackages = await getThirdPartyPackages(); - let languages = hljs.listLanguages(); + const languages = hljs.listLanguages(); describe(`hljs.highlightAuto()`, function() { languages.filter(hljs.autoDetection).forEach((language) => { - let detectPath = detectTestDir(language); + const detectPath = detectTestDir(language); testAutoDetection(language, { detectPath }); }); }); @@ -50,13 +48,11 @@ describe('hljs.highlightAuto()', () => { for (let i = 0; i < thirdPartyPackages.length; ++i) { const pkg = thirdPartyPackages[i]; const idx = pkg.names.indexOf(name); - if (idx !== -1) - return pkg.detectTestPaths[idx] + if (idx !== -1) return pkg.detectTestPaths[idx]; } return null; // test not found } }); - it("adding dynamic tests...", async function() {} ); // this is required to work + it("adding dynamic tests...", async function() {}); // this is required to work }); - diff --git a/test/markup/coffeescript/function.expect.txt b/test/markup/coffeescript/function.expect.txt index 0c4f8af86f..4d43e641b3 100644 --- a/test/markup/coffeescript/function.expect.txt +++ b/test/markup/coffeescript/function.expect.txt @@ -3,12 +3,12 @@ square = (x) -> x * x npmWishlist.sha256 = (str) -> - throw new Error() + throw new Error() str.split(" ").map((m) -> m.charCodeAt(0)) fs.readFile("package.json", "utf-8", (err, content) -> - data = JSON.parse(content) + data = JSON.parse(content) data.version ) diff --git a/test/markup/javascript/class.expect.txt b/test/markup/javascript/class.expect.txt index 31a8507e4c..c631c8f819 100644 --- a/test/markup/javascript/class.expect.txt +++ b/test/markup/javascript/class.expect.txt @@ -1,11 +1,11 @@ class Car extends Vehicle { constructor(speed, cost) { - super(speed); + super(speed); var c = Symbol('cost'); - this[c] = cost; + this[c] = cost; - this.intro = `This is a car runs at + this.intro = `This is a car runs at ${speed}.`; } } diff --git a/test/markup/typescript/class.expect.txt b/test/markup/typescript/class.expect.txt index ff99191bf4..41894432d7 100644 --- a/test/markup/typescript/class.expect.txt +++ b/test/markup/typescript/class.expect.txt @@ -1,11 +1,11 @@ class Car extends Vehicle { constructor(speed, cost) { - super(speed); + super(speed); - var c = Symbol('cost'); - this[c] = cost; + var c = Symbol('cost'); + this[c] = cost; - this.intro = `This is a car runs at + this.intro = `This is a car runs at ${speed}.`; } }