enh(fsharp) myriad of fixes and improvements (#3357)

- fix: Stop highlighting type names everywhere. Only match them in type annotations. - Since the names are not reserved and can be re-used for variable/binding names shadowing, it can be confusing and misleading. - enh: Highlight operators. Operators are a big part of F#, so highlighting them brings a lot of value for legibility, etc... - enh: Support quoted identifiers (` ``...`` `. They can contain pretty much anything inside the double backquotes) - fix: Fix type names and symbols matching * Type names can contain apostrophes. * Type symbols can contain quoted identifiers, but no apostrophes. - enh: Match attributes in more places (not just at the start of a line), and match more of the things they can contain (not just basic strings and numbers).
highlightjs · Nov 29, 2021 · 238e073 · 238e073
1 parent 59c02d1
commit 238e073
Show file tree

Hide file tree

Showing 16 changed files with 605 additions and 186 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -5,6 +5,7 @@ Grammars:
 - fix(python) def, class keywords detected mid-identifier (#3381) [Josh Goebel][]
 - fix(python) Fix recognition of numeric literals followed by keywords without whitespace (#2985) [Richard Gibson][]
 - enh(swift) add SE-0290 unavailability condition (#3382) [Bradley Mackey][]
+- fix(fsharp) Highlight operators, match type names only in type annotations, support quoted identifiers, and other smaller fixes. [Melvyn Laïly][]
 - enh(java) add `sealed` and `non-sealed` keywords (#3386) [Bradley Mackey][]
 - fix(clojure) Several issues with Clojure highlighting (#3397) [Björn Ebbinghaus][]
   - fix(clojure) `comment` macro catches more than it should (#3395)
@@ -22,6 +23,7 @@ Developer Tools:
 
 [Richard Gibson]: https://github.com/gibson042
 [Bradley Mackey]: https://github.com/bradleymackey
+[Melvyn Laïly]: https://github.com/mlaily
 [Björn Ebbinghaus]: https://github.com/MrEbbinghaus
 [Josh Goebel]: https://github.com/joshgoebel
 

diff --git a/src/languages/fsharp.js b/src/languages/fsharp.js
@@ -119,7 +119,9 @@ export default function(hljs) {
     "__SOURCE_FILE__"
   ];
 
-  const TYPES = [
+  // Since it's possible to re-bind/shadow names (e.g. let char = 'c'),
+  // these builtin types should only be matched when a type name is expected.
+  const KNOWN_TYPES = [
     // basic types
     "bool",
     "byte",
@@ -157,7 +159,9 @@ export default function(hljs) {
     "nativeptr",
     "obj",
     "outref",
-    "voidptr"
+    "voidptr",
+    // other important FSharp types
+    "Result"
   ];
 
   const BUILTINS = [
@@ -172,6 +176,7 @@ export default function(hljs) {
     "dict",
     "readOnlyDict",
     "set",
+    "get",
     "enum",
     "sizeof",
     "typeof",
@@ -201,7 +206,6 @@ export default function(hljs) {
   ];
 
   const ALL_KEYWORDS = {
-    type: TYPES,
     keyword: KEYWORDS,
     literal: LITERALS,
     built_in: BUILTINS,
@@ -221,16 +225,138 @@ export default function(hljs) {
     ]
   };
 
-  // 'a or ^a
+  // Most identifiers can contain apostrophes
+  const IDENTIFIER_RE = /[a-zA-Z_](\w|')*/;
+
+  const QUOTED_IDENTIFIER = {
+    scope: 'variable',
+    begin: /``/,
+    end: /``/
+  };
+
+  // 'a or ^a where a can be a ``quoted identifier``
+  const BEGIN_GENERIC_TYPE_SYMBOL_RE = /\B('|\^)/
   const GENERIC_TYPE_SYMBOL = {
-    match: regex.concat(/('|\^)/, hljs.UNDERSCORE_IDENT_RE),
     scope: 'symbol',
+    variants: [
+      // the type name is a quoted identifier:
+      { match: regex.concat(BEGIN_GENERIC_TYPE_SYMBOL_RE, /``.*?``/) },
+      // the type name is a normal identifier (we don't use IDENTIFIER_RE because there cannot be another apostrophe here):
+      { match: regex.concat(BEGIN_GENERIC_TYPE_SYMBOL_RE, hljs.UNDERSCORE_IDENT_RE) }
+    ],
     relevance: 0
   };
 
+  const makeOperatorMode = function({ includeEqual }) {
+    // List or symbolic operator characters from the FSharp Spec 4.1, minus the dot, and with `?` added, used for nullable operators.
+    let allOperatorChars;
+    if (includeEqual)
+      allOperatorChars = "!%&*+-/<=>@^|~?";
+    else
+      allOperatorChars = "!%&*+-/<>@^|~?";
+    const OPERATOR_CHARS = Array.from(allOperatorChars);
+    const OPERATOR_CHAR_RE = regex.concat('[', ...OPERATOR_CHARS.map(regex.escape), ']');
+    // The lone dot operator is special. It cannot be redefined, and we don't want to highlight it. It can be used as part of a multi-chars operator though.
+    const OPERATOR_CHAR_OR_DOT_RE = regex.either(OPERATOR_CHAR_RE, /\./);
+    // When a dot is present, it must be followed by another operator char:
+    const OPERATOR_FIRST_CHAR_OF_MULTIPLE_RE = regex.concat(OPERATOR_CHAR_OR_DOT_RE, regex.lookahead(OPERATOR_CHAR_OR_DOT_RE));
+    const SYMBOLIC_OPERATOR_RE = regex.either(
+      regex.concat(OPERATOR_FIRST_CHAR_OF_MULTIPLE_RE, OPERATOR_CHAR_OR_DOT_RE, '*'), // Matches at least 2 chars operators
+      regex.concat(OPERATOR_CHAR_RE, '+'), // Matches at least one char operators
+    );
+    return {
+      scope: 'operator',
+      match: regex.either(
+        // symbolic operators:
+        SYMBOLIC_OPERATOR_RE,
+        // other symbolic keywords:
+        // Type casting and conversion operators:
+        /:\?>/,
+        /:\?/,
+        /:>/,
+        /:=/, // Reference cell assignment
+        /::?/, // : or ::
+        /\$/), // A single $ can be used as an operator
+      relevance: 0
+    };
+  }
+
+  const OPERATOR = makeOperatorMode({ includeEqual: true });
+  // This variant is used when matching '=' should end a parent mode:
+  const OPERATOR_WITHOUT_EQUAL = makeOperatorMode({ includeEqual: false });
+
+  const makeTypeAnnotationMode = function(prefix, prefixScope) {
+    return {
+      begin: regex.concat( // a type annotation is a
+        prefix,            // should be a colon or the 'of' keyword
+        regex.lookahead(   // that has to be followed by
+          regex.concat(
+            /\s*/,         // optional space
+            regex.either(  // then either of:
+              /\w/,        // word
+              /'/,         // generic type name
+              /\^/,        // generic type name
+              /#/,         // flexible type name
+              /``/,        // quoted type name
+              /\(/,        // parens type expression
+              /{\|/,       // anonymous type annotation
+      )))),
+      beginScope: prefixScope,
+      // BUG: because ending with \n is necessary for some cases, multi-line type annotations are not properly supported.
+      // Examples where \n is required at the end:
+      // - abstract member definitions in classes: abstract Property : int * string
+      // - return type annotations: let f f' = f' () : returnTypeAnnotation
+      // - record fields definitions: { A : int \n B : string }
+      end: regex.lookahead(
+        regex.either(
+          /\n/,
+          /=/)),
+      relevance: 0,
+      // we need the known types, and we need the type constraint keywords and literals. e.g.: when 'a : null
+      keywords: hljs.inherit(ALL_KEYWORDS, { type: KNOWN_TYPES }),
+      contains: [
+        COMMENT,
+        GENERIC_TYPE_SYMBOL,
+        hljs.inherit(QUOTED_IDENTIFIER, { scope: null }), // match to avoid strange patterns inside that may break the parsing
+        OPERATOR_WITHOUT_EQUAL
+      ]
+    };
+  }
+
+  const TYPE_ANNOTATION = makeTypeAnnotationMode(/:/, 'operator');
+  const DISCRIMINATED_UNION_TYPE_ANNOTATION = makeTypeAnnotationMode(/\bof\b/, 'keyword');
+
+  // type MyType<'a> = ...
+  const TYPE_DECLARATION = {
+    begin: [
+      /(^|\s+)/, // prevents matching the following: `match s.stype with`
+      /type/,
+      /\s+/,
+      IDENTIFIER_RE
+    ],
+    beginScope: {
+      2: 'keyword',
+      4: 'title.class'
+    },
+    end: regex.lookahead(/\(|=|$/),
+    keywords: ALL_KEYWORDS, // match keywords in type constraints. e.g.: when 'a : null
+    contains: [
+      COMMENT,
+      hljs.inherit(QUOTED_IDENTIFIER, { scope: null }), // match to avoid strange patterns inside that may break the parsing
+      GENERIC_TYPE_SYMBOL,
+      {
+        // For visual consistency, highlight type brackets as operators.
+        scope: 'operator',
+        match: /<|>/
+      },
+      TYPE_ANNOTATION // generic types can have constraints, which are type annotations. e.g. type MyType<'T when 'T : delegate<obj * string>> =
+    ]
+  };
+
   const COMPUTATION_EXPRESSION = {
     // computation expressions:
     scope: 'computation-expression',
+    // BUG: might conflict with record deconstruction. e.g. let f { Name = name } = name // will highlight f
     match: /\b[_a-z]\w*(?=\s*\{)/
   };
 
@@ -365,10 +491,13 @@ export default function(hljs) {
     CHAR_LITERAL,
     BANG_KEYWORD_MODE,
     COMMENT,
+    QUOTED_IDENTIFIER,
+    TYPE_ANNOTATION,
     COMPUTATION_EXPRESSION,
     PREPROCESSOR,
     NUMBER,
-    GENERIC_TYPE_SYMBOL
+    GENERIC_TYPE_SYMBOL,
+    OPERATOR
   ];
   const STRING = {
     variants: [
@@ -397,42 +526,32 @@ export default function(hljs) {
       BANG_KEYWORD_MODE,
       STRING,
       COMMENT,
+      QUOTED_IDENTIFIER,
+      TYPE_DECLARATION,
       {
-        // type MyType<'a> = ...
-        begin: [
-          /type/,
-          /\s+/,
-          hljs.UNDERSCORE_IDENT_RE
-        ],
-        beginScope: {
-          1: 'keyword',
-          3: 'title.class'
-        },
-        end: regex.lookahead(/\(|=|$/),
-        contains: [
-          GENERIC_TYPE_SYMBOL
-        ]
-      },
-      {
-        // [<Attributes("")>]
+        // e.g. [<Attributes("")>] or [<``module``: MyCustomAttributeThatWorksOnModules>]
+        // or [<Sealed; NoEquality; NoComparison; CompiledName("FSharpAsync`1")>]
         scope: 'meta',
-        begin: /^\s*\[</,
-        excludeBegin: true,
-        end: regex.lookahead(/>\]/),
+        begin: /\[</,
+        end: />\]/,
         relevance: 2,
         contains: [
-          {
-            scope: 'string',
-            begin: /"/,
-            end: /"/
-          },
+          QUOTED_IDENTIFIER,
+          // can contain any constant value
+          TRIPLE_QUOTED_STRING,
+          VERBATIM_STRING,
+          QUOTED_STRING,
+          CHAR_LITERAL,
           NUMBER
         ]
       },
+      DISCRIMINATED_UNION_TYPE_ANNOTATION,
+      TYPE_ANNOTATION,
       COMPUTATION_EXPRESSION,
       PREPROCESSOR,
       NUMBER,
-      GENERIC_TYPE_SYMBOL
+      GENERIC_TYPE_SYMBOL,
+      OPERATOR
     ]
   };
 }
diff --git a/test/markup/fsharp/attributes.expect.txt b/test/markup/fsharp/attributes.expect.txt
@@ -1,4 +1,4 @@
 <span class="hljs-comment">// Strings and numbers are highlighted inside the attribute</span>
-[&lt;<span class="hljs-meta">Foo</span>&gt;]
-[&lt;<span class="hljs-meta">Bar(<span class="hljs-string">&quot;bar&quot;</span>); Foo(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>)</span>&gt;]
-<span class="hljs-keyword">let</span> x () = ()
+<span class="hljs-meta">[&lt;Foo&gt;]</span>
+<span class="hljs-meta">[&lt;Bar(<span class="hljs-string">&quot;bar&quot;</span>); Foo(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>)&gt;]</span>
+<span class="hljs-keyword">let</span> x () <span class="hljs-operator">=</span> ()
diff --git a/test/markup/fsharp/bang-keywords.expect.txt b/test/markup/fsharp/bang-keywords.expect.txt
@@ -1 +1 @@
-<span class="hljs-keyword">let!</span> (result2 : <span class="hljs-type">byte</span>[])  = stream.AsyncRead(bufferSize)
+<span class="hljs-keyword">let!</span> (result2 <span class="hljs-operator">:</span> <span class="hljs-type">byte</span>[]) <span class="hljs-operator">=</span> stream.AsyncRead(bufferSize)
diff --git a/test/markup/fsharp/bang-keywords.txt b/test/markup/fsharp/bang-keywords.txt
@@ -1 +1 @@
-let! (result2 : byte[])  = stream.AsyncRead(bufferSize)
+let! (result2 : byte[]) = stream.AsyncRead(bufferSize)
diff --git a/test/markup/fsharp/comments.expect.txt b/test/markup/fsharp/comments.expect.txt
@@ -14,12 +14,12 @@ asdf
 
 *)</span>
 
-<span class="hljs-keyword">let</span> index =
+<span class="hljs-keyword">let</span> index <span class="hljs-operator">=</span>
     len
-    |&gt; <span class="hljs-type">float</span>
-    |&gt; Operators.(*) <span class="hljs-number">0.1</span>      <span class="hljs-comment">// (*) here is not comment</span>
-    |&gt; Operators.(+) <span class="hljs-number">1</span>        <span class="hljs-comment">// (+) here is not comment</span>
-    |&gt; Operators.(-) len      <span class="hljs-comment">// (-) here is not comment</span>
+    <span class="hljs-operator">|&gt;</span> float
+    <span class="hljs-operator">|&gt;</span> Operators.(<span class="hljs-operator">*</span>) <span class="hljs-number">0.1</span>      <span class="hljs-comment">// (*) here is not comment</span>
+    <span class="hljs-operator">|&gt;</span> Operators.(<span class="hljs-operator">+</span>) <span class="hljs-number">1</span>        <span class="hljs-comment">// (+) here is not comment</span>
+    <span class="hljs-operator">|&gt;</span> Operators.(<span class="hljs-operator">-</span>) len      <span class="hljs-comment">// (-) here is not comment</span>
 
 
 <span class="hljs-comment">// foobar</span>
@@ -34,11 +34,11 @@ asdf
 <span class="hljs-comment">/// Longer comments can be associated with a type or member through</span>
 <span class="hljs-comment">/// the remarks tag.</span>
 <span class="hljs-comment">/// &lt;/remarks&gt;</span>
-<span class="hljs-keyword">let</span> x = ()
+<span class="hljs-keyword">let</span> x <span class="hljs-operator">=</span> ()
 
 <span class="hljs-comment">// the next one is not a comment</span>
-(*) (*)
+(<span class="hljs-operator">*</span>) (<span class="hljs-operator">*</span>)
 
-/*
+<span class="hljs-operator">/*</span>
 this one is <span class="hljs-built_in">not</span> a valid comment either
-*/
+<span class="hljs-operator">*/</span>
diff --git a/test/markup/fsharp/computation-expressions.expect.txt b/test/markup/fsharp/computation-expressions.expect.txt
@@ -3,21 +3,21 @@
 <span class="hljs-keyword">open</span> System.Threading.Tasks
 
 <span class="hljs-comment">// Single line, and contains a capital letter</span>
-<span class="hljs-keyword">let</span> unitTask = <span class="hljs-keyword">unitTask</span> { <span class="hljs-keyword">return</span> () }
+<span class="hljs-keyword">let</span> unitTask <span class="hljs-operator">=</span> <span class="hljs-keyword">unitTask</span> { <span class="hljs-keyword">return</span> () }
 
-<span class="hljs-keyword">let</span> work =
+<span class="hljs-keyword">let</span> work <span class="hljs-operator">=</span>
     <span class="hljs-keyword">async</span> {
-        <span class="hljs-keyword">let</span> delayTask () =
+        <span class="hljs-keyword">let</span> delayTask () <span class="hljs-operator">=</span>
             <span class="hljs-comment">// Nested computation</span>
             <span class="hljs-keyword">task</span> {
                 <span class="hljs-built_in">printfn</span> <span class="hljs-string">&quot;Delay...&quot;</span>
                 <span class="hljs-keyword">do!</span> Task.Delay <span class="hljs-number">1000</span>
                 <span class="hljs-keyword">return</span> <span class="hljs-number">42</span>
             }
-        <span class="hljs-keyword">let!</span> result = delayTask () |&gt; Async.AwaitTask
+        <span class="hljs-keyword">let!</span> result <span class="hljs-operator">=</span> delayTask () <span class="hljs-operator">|&gt;</span> Async.AwaitTask
         <span class="hljs-built_in">printfn</span> <span class="hljs-string">&quot;Async F# sleep...&quot;</span>
         <span class="hljs-keyword">do!</span> Async.Sleep <span class="hljs-number">1000</span>
         <span class="hljs-keyword">return</span> result
     }
 
-<span class="hljs-keyword">let</span> result = work |&gt; Async.RunSynchronously
+<span class="hljs-keyword">let</span> result <span class="hljs-operator">=</span> work <span class="hljs-operator">|&gt;</span> Async.RunSynchronously
diff --git a/test/markup/fsharp/fsi-and-preprocessor.expect.txt b/test/markup/fsharp/fsi-and-preprocessor.expect.txt
@@ -7,16 +7,16 @@
 <span class="hljs-meta">#nowarn</span>
 
   <span class="hljs-meta">#if</span> DEBUG <span class="hljs-comment">// whitespace is allowed before</span>
-<span class="hljs-keyword">let</span> x = <span class="hljs-number">0</span> #<span class="hljs-keyword">if</span> DEBUG <span class="hljs-comment">// but the preprocessor directive must be the first non-whitespace</span>
+<span class="hljs-keyword">let</span> x <span class="hljs-operator">=</span> <span class="hljs-number">0</span> #<span class="hljs-keyword">if</span> DEBUG <span class="hljs-comment">// but the preprocessor directive must be the first non-whitespace</span>
 
 #IF asdf <span class="hljs-comment">// should not match wrongly cased keywords</span>
 #iftest <span class="hljs-comment">// should not match</span>
 
 <span class="hljs-meta">#r</span> <span class="hljs-string">&quot;file.dll&quot;</span>;;                               <span class="hljs-comment">// Reference (dynamically load) the given DLL</span>
 <span class="hljs-meta">#i</span> <span class="hljs-string">&quot;package source uri&quot;</span>;;                     <span class="hljs-comment">// Include package source uri when searching for packages</span>
 <span class="hljs-meta">#I</span> <span class="hljs-string">&quot;path&quot;</span>;;                                   <span class="hljs-comment">// Add the given search path for referenced DLLs</span>
-<span class="hljs-meta">#load</span> <span class="hljs-string">&quot;file.fs&quot;</span> ...;;                         <span class="hljs-comment">// Load the given file(s) as if compiled and referenced</span>
-<span class="hljs-meta">#time</span> [<span class="hljs-string">&quot;on&quot;</span>|<span class="hljs-string">&quot;off&quot;</span>];;                          <span class="hljs-comment">// Toggle timing on/off</span>
+<span class="hljs-meta">#load</span> <span class="hljs-string">&quot;file.fs&quot;</span> <span class="hljs-operator">...</span>;;                         <span class="hljs-comment">// Load the given file(s) as if compiled and referenced</span>
+<span class="hljs-meta">#time</span> [<span class="hljs-string">&quot;on&quot;</span><span class="hljs-operator">|</span><span class="hljs-string">&quot;off&quot;</span>];;                          <span class="hljs-comment">// Toggle timing on/off</span>
 <span class="hljs-meta">#help</span>;;                                       <span class="hljs-comment">// Display help</span>
 <span class="hljs-meta">#r</span> <span class="hljs-string">&quot;nuget:FSharp.Data, 3.1.2&quot;</span>;;               <span class="hljs-comment">// Load Nuget Package &#x27;FSharp.Data&#x27; version &#x27;3.1.2&#x27;</span>
 <span class="hljs-meta">#r</span> <span class="hljs-string">&quot;nuget:FSharp.Data&quot;</span>;;                      <span class="hljs-comment">// Load Nuget Package &#x27;FSharp.Data&#x27; with the highest version</span>