fix: Multiple Partiql and Amazon Ion textual notation fixes (ajaxorg#…

…4686) * Bug fixes & improvements to Ion mode - Changes symbols to `storage` type instead of `constant` - Bug fix for some edge cases around annotations - Bug fixes for some edge cases around strings better * Bug fixes & improvements to PartiQL mode - Rearranges rule order so punctuation fallback doesn't clobber collections - Bug fixes for collection types - Bug fixes for some edge cases around strings better - Bug fix for embedded ion rule end token type
jeroenpost86 · May 13, 2022 · bffba8d · bffba8d
1 parent 33cf1c6
commit bffba8d
Show file tree

Hide file tree

Showing 7 changed files with 221 additions and 184 deletions.
diff --git a/demo/kitchen-sink/docs/ion.ion b/demo/kitchen-sink/docs/ion.ion
@@ -7,7 +7,7 @@ Comment
 [1,2,3,45 /*TODO: should be 42, obviously*/, 2016-04-21T08:10:46Z, 2016-04-21T08:10:46-08:00, 2016-04-21, 2016, 0x5, 0b0110]
 (5 7 1 3 [
     'a', 'b',
-    null, null.int, null.bool, null.list
+    null, null.int, null.bool, null.list,
     true, false,
     nan, -inf, +inf,
     "str",
@@ -33,15 +33,15 @@ states::{
 
     // https://amzn.github.io/ion-docs/docs/spec.html#string
     string: rules::[
-        match::{
+        match :: {
             regex: "(\\\")((?:\\\\\"|[^\"])*)(\\\")",
             token: ['string.dblq.punc.start', 'string.dblq', 'string.dblq.punc.end']
         },
         push::{
             regex: "\'{3}",
             token: 'string.trpq.punc.start',
             states: rules::[
-                pop::{
+                pop     ::     {
                     regex: "\'{3}",
                     token: 'string.trpq.punc.end',
                 },
@@ -58,7 +58,7 @@ states::{
     */
 }
 {
- "json": {
+"json": {
       "compatibility": true,
       "open sourced": "2016-04-21T08:10:46Z",
       "foo": 9,

diff --git a/demo/kitchen-sink/docs/partiql.partiql b/demo/kitchen-sink/docs/partiql.partiql
@@ -3,9 +3,8 @@
    line
    comment */
 
-SELECT a, b, c FROM stuff s INNER CROSS JOIN @s WHERE f(s)  -- comment
+SELECT "a", b, c FROM stuff s INNER CROSS JOIN @s WHERE f(s)  -- comment
 -- comment
-SELECT "a", b FROM stuff s, @s WHERE f(s)
 
 SELECT VALUE {'sensor': s.sensor,
               'readings': (SELECT VALUE l.co
@@ -30,6 +29,7 @@ FROM `[{'a':1, 'b':1}, {'a':2}, "foo"]` AS x
 
 SELECT VALUE {v.a: v.b, v.c: v.d}
 FROM <<{'a':'same', 'b':1, 'c':'same', 'd':2}>> AS v
+WHERE v.b
 
 SELECT u.id, feedbackId, commentId, upvoteId
 FROM users as u, u.feedbacks as feedback at feedbackId
@@ -52,9 +52,6 @@ SELECT (
 
 SELECT SUM(AVG(n)) FROM <<numbers, numbers>> AS n
 
-SELECT VALUES v.a
-FROM [{'a':1, 'b':true}, {'a':2, 'b':null}, {'a':3}] v
-WHERE v.b
 
 SELECT attributeId, COUNT(*) as the_count
 FROM repeating_things

diff --git a/lib/ace/mode/_test/tokens_ion.json b/lib/ace/mode/_test/tokens_ion.json
@@ -62,10 +62,10 @@
 ],[
    ["punctuation.definition.list.begin.ion","punctuation.definition.sexp.begin.ion"],
   ["text.ion","    "],
-  ["constant.other.symbol.quoted.ion","'a'"],
+  ["storage.type.symbol.quoted.ion","'a'"],
   ["punctuation.definition.list.separator.ion",","],
   ["text.ion"," "],
-  ["constant.other.symbol.quoted.ion","'b'"],
+  ["storage.type.symbol.quoted.ion","'b'"],
   ["punctuation.definition.list.separator.ion",","]
 ],[
    ["punctuation.definition.list.begin.ion","punctuation.definition.sexp.begin.ion"],
@@ -163,7 +163,8 @@
    "start"
 ],[
    "punctuation.definition.struct.begin.ion",
-  ["variable.language.annotation.ion","states::"],
+  ["variable.language.annotation.ion","states"],
+  ["punctuation.definition.annotation.ion","::"],
   ["punctuation.definition.struct.begin.ion","{"]
 ],[
    "punctuation.definition.struct.begin.ion",
@@ -175,13 +176,15 @@
   ["constant.other.symbol.identifier.ion","root"],
   ["punctuation.definition.struct.separator.ion",":"],
   ["text.ion"," "],
-  ["variable.language.annotation.ion","rules::"],
+  ["variable.language.annotation.ion","rules"],
+  ["punctuation.definition.annotation.ion","::"],
   ["punctuation.definition.list.begin.ion","["]
 ],[
    ["punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
   ["text.ion","        "],
-  ["variable.language.annotation.ion","include::"],
-  ["constant.other.symbol.quoted.ion","'value'"]
+  ["variable.language.annotation.ion","include"],
+  ["punctuation.definition.annotation.ion","::"],
+  ["storage.type.symbol.quoted.ion","'value'"]
 ],[
    "punctuation.definition.struct.begin.ion",
   ["text.ion","    "],
@@ -199,12 +202,14 @@
   ["constant.other.symbol.identifier.ion","string"],
   ["punctuation.definition.struct.separator.ion",":"],
   ["text.ion"," "],
-  ["variable.language.annotation.ion","rules::"],
+  ["variable.language.annotation.ion","rules"],
+  ["punctuation.definition.annotation.ion","::"],
   ["punctuation.definition.list.begin.ion","["]
 ],[
    ["punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
   ["text.ion","        "],
-  ["variable.language.annotation.ion","match::"],
+  ["variable.language.annotation.ion","match"],
+  ["punctuation.definition.annotation.ion","::"],
   ["punctuation.definition.struct.begin.ion","{"]
 ],[
    ["punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
@@ -223,13 +228,13 @@
   ["punctuation.definition.struct.separator.ion",":"],
   ["text.ion"," "],
   ["punctuation.definition.list.begin.ion","["],
-  ["constant.other.symbol.quoted.ion","'string.dblq.punc.start'"],
+  ["storage.type.symbol.quoted.ion","'string.dblq.punc.start'"],
   ["punctuation.definition.list.separator.ion",","],
   ["text.ion"," "],
-  ["constant.other.symbol.quoted.ion","'string.dblq'"],
+  ["storage.type.symbol.quoted.ion","'string.dblq'"],
   ["punctuation.definition.list.separator.ion",","],
   ["text.ion"," "],
-  ["constant.other.symbol.quoted.ion","'string.dblq.punc.end'"],
+  ["storage.type.symbol.quoted.ion","'string.dblq.punc.end'"],
   ["punctuation.definition.list.end.ion","]"]
 ],[
    ["punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
@@ -239,7 +244,8 @@
 ],[
    ["punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
   ["text.ion","        "],
-  ["variable.language.annotation.ion","push::"],
+  ["variable.language.annotation.ion","push"],
+  ["punctuation.definition.annotation.ion","::"],
   ["punctuation.definition.struct.begin.ion","{"]
 ],[
    ["punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
@@ -257,20 +263,22 @@
   ["constant.other.symbol.identifier.ion","token"],
   ["punctuation.definition.struct.separator.ion",":"],
   ["text.ion"," "],
-  ["constant.other.symbol.quoted.ion","'string.trpq.punc.start'"],
+  ["storage.type.symbol.quoted.ion","'string.trpq.punc.start'"],
   ["punctuation.definition.struct.separator.ion",","]
 ],[
    ["punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
   ["text.ion","            "],
   ["constant.other.symbol.identifier.ion","states"],
   ["punctuation.definition.struct.separator.ion",":"],
   ["text.ion"," "],
-  ["variable.language.annotation.ion","rules::"],
+  ["variable.language.annotation.ion","rules"],
+  ["punctuation.definition.annotation.ion","::"],
   ["punctuation.definition.list.begin.ion","["]
 ],[
    ["punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
   ["text.ion","                "],
-  ["variable.language.annotation.ion","pop::"],
+  ["variable.language.annotation.ion","pop"],
+  ["punctuation.definition.annotation.ion","::"],
   ["punctuation.definition.struct.begin.ion","{"]
 ],[
    ["punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
@@ -288,7 +296,7 @@
   ["constant.other.symbol.identifier.ion","token"],
   ["punctuation.definition.struct.separator.ion",":"],
   ["text.ion"," "],
-  ["constant.other.symbol.quoted.ion","'string.trpq.punc.end'"],
+  ["storage.type.symbol.quoted.ion","'string.trpq.punc.end'"],
   ["punctuation.definition.struct.separator.ion",","]
 ],[
    ["punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
@@ -298,7 +306,8 @@
 ],[
    ["punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
   ["text.ion","                "],
-  ["variable.language.annotation.ion","match::"],
+  ["variable.language.annotation.ion","match"],
+  ["punctuation.definition.annotation.ion","::"],
   ["punctuation.definition.struct.begin.ion","{"]
 ],[
    ["punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
@@ -316,7 +325,7 @@
   ["constant.other.symbol.identifier.ion","token"],
   ["punctuation.definition.struct.separator.ion",":"],
   ["text.ion"," "],
-  ["constant.other.symbol.quoted.ion","'string.trpq'"]
+  ["storage.type.symbol.quoted.ion","'string.trpq'"]
 ],[
    ["punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.struct.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.list.begin.ion","punctuation.definition.struct.begin.ion"],
   ["text.ion","                "],