Add Python 3.8 features.

Add walrus operator `:=`. Add positional only args separator `/`. Add `=` to f-string formatting. Fixes #189
MagicStack · Oct 14, 2022 · db32a0d · db32a0d
1 parent 2802ded
commit db32a0d
Show file tree

Hide file tree

Showing 11 changed files with 439 additions and 18 deletions.
diff --git a/grammars/MagicPython.cson b/grammars/MagicPython.cson
@@ -619,6 +619,8 @@ repository:
       
           | (!= | == | >= | <= | < | >)                          (?# 5)
       
+          | (:=)                                                 (?# 6)
+      
     '''
     captures:
       "1":
@@ -631,6 +633,8 @@ repository:
         name: "keyword.operator.arithmetic.python"
       "5":
         name: "keyword.operator.comparison.python"
+      "6":
+        name: "keyword.operator.assignment.python"
   punctuation:
     patterns: [
       {
@@ -1255,6 +1259,10 @@ repository:
             name: "punctuation.section.function.lambda.begin.python"
         contentName: "meta.function.lambda.parameters.python"
         patterns: [
+          {
+            name: "keyword.operator.positional.parameter.python"
+            match: "/"
+          }
           {
             name: "keyword.operator.unpacking.parameter.python"
             match: "(\\*\\*|\\*)"
@@ -1402,6 +1410,10 @@ repository:
       "1":
         name: "punctuation.definition.parameters.end.python"
     patterns: [
+      {
+        name: "keyword.operator.positional.parameter.python"
+        match: "/"
+      }
       {
         name: "keyword.operator.unpacking.parameter.python"
         match: "(\\*\\*|\\*)"
@@ -4720,12 +4732,16 @@ repository:
     patterns: [
       {
         name: "storage.type.format.python"
-        match: "(![rsa])(?=})"
+        match: "(=(![rsa])?)(?=})"
+      }
+      {
+        name: "storage.type.format.python"
+        match: "(=?![rsa])(?=})"
       }
       {
         match: '''
           (?x)
-            (![rsa])?
+            ( (?: =?) (?: ![rsa])? )
               ( : \\w? [<>=^]? [-+ ]? \\#?
                 \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )(?=})
           
@@ -4741,7 +4757,7 @@ repository:
       }
     ]
   "fstring-terminator-single-tail":
-    begin: "(![rsa])?(:)(?=.*?{)"
+    begin: "((?:=?)(?:![rsa])?)(:)(?=.*?{)"
     end: "(?=})|(?=\\n)"
     beginCaptures:
       "1":
@@ -4944,12 +4960,16 @@ repository:
     patterns: [
       {
         name: "storage.type.format.python"
-        match: "(![rsa])(?=})"
+        match: "(=(![rsa])?)(?=})"
+      }
+      {
+        name: "storage.type.format.python"
+        match: "(=?![rsa])(?=})"
       }
       {
         match: '''
           (?x)
-            (![rsa])?
+            ( (?: =?) (?: ![rsa])? )
               ( : \\w? [<>=^]? [-+ ]? \\#?
                 \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )(?=})
           
@@ -4965,7 +4985,7 @@ repository:
       }
     ]
   "fstring-terminator-multi-tail":
-    begin: "(![rsa])?(:)(?=.*?{)"
+    begin: "((?:=?)(?:![rsa])?)(:)(?=.*?{)"
     end: "(?=})"
     beginCaptures:
       "1":

diff --git a/grammars/MagicPython.tmLanguage b/grammars/MagicPython.tmLanguage
@@ -933,6 +933,8 @@ E.g. "arr[idx](args)"
     | (\*\* | \* | \+ | - | % | // | / | @)                (?# 4)
 
     | (!= | == | &gt;= | &lt;= | &lt; | &gt;)                          (?# 5)
+
+    | (:=)                                                 (?# 6)
 </string>
         <key>captures</key>
         <dict>
@@ -961,6 +963,11 @@ E.g. "arr[idx](args)"
             <key>name</key>
             <string>keyword.operator.comparison.python</string>
           </dict>
+          <key>6</key>
+          <dict>
+            <key>name</key>
+            <string>keyword.operator.assignment.python</string>
+          </dict>
         </dict>
       </dict>
       <key>punctuation</key>
@@ -1936,6 +1943,12 @@ E.g. "arr[idx](args)"
             <string>meta.function.lambda.parameters.python</string>
             <key>patterns</key>
             <array>
+              <dict>
+                <key>name</key>
+                <string>keyword.operator.positional.parameter.python</string>
+                <key>match</key>
+                <string>/</string>
+              </dict>
               <dict>
                 <key>name</key>
                 <string>keyword.operator.unpacking.parameter.python</string>
@@ -2184,6 +2197,12 @@ correctly identify the "in" as a control flow keyword.
         </dict>
         <key>patterns</key>
         <array>
+          <dict>
+            <key>name</key>
+            <string>keyword.operator.positional.parameter.python</string>
+            <key>match</key>
+            <string>/</string>
+          </dict>
           <dict>
             <key>name</key>
             <string>keyword.operator.unpacking.parameter.python</string>
@@ -8259,12 +8278,18 @@ indirectly through syntactic constructs
             <key>name</key>
             <string>storage.type.format.python</string>
             <key>match</key>
-            <string>(![rsa])(?=})</string>
+            <string>(=(![rsa])?)(?=})</string>
+          </dict>
+          <dict>
+            <key>name</key>
+            <string>storage.type.format.python</string>
+            <key>match</key>
+            <string>(=?![rsa])(?=})</string>
           </dict>
           <dict>
             <key>match</key>
             <string>(?x)
-  (![rsa])?
+  ( (?: =?) (?: ![rsa])? )
     ( : \w? [&lt;&gt;=^]? [-+ ]? \#?
       \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )(?=})
 </string>
@@ -8291,7 +8316,7 @@ indirectly through syntactic constructs
       <key>fstring-terminator-single-tail</key>
       <dict>
         <key>begin</key>
-        <string>(![rsa])?(:)(?=.*?{)</string>
+        <string>((?:=?)(?:![rsa])?)(:)(?=.*?{)</string>
         <key>end</key>
         <string>(?=})|(?=\n)</string>
         <key>beginCaptures</key>
@@ -8631,12 +8656,18 @@ indirectly through syntactic constructs
             <key>name</key>
             <string>storage.type.format.python</string>
             <key>match</key>
-            <string>(![rsa])(?=})</string>
+            <string>(=(![rsa])?)(?=})</string>
+          </dict>
+          <dict>
+            <key>name</key>
+            <string>storage.type.format.python</string>
+            <key>match</key>
+            <string>(=?![rsa])(?=})</string>
           </dict>
           <dict>
             <key>match</key>
             <string>(?x)
-  (![rsa])?
+  ( (?: =?) (?: ![rsa])? )
     ( : \w? [&lt;&gt;=^]? [-+ ]? \#?
       \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )(?=})
 </string>
@@ -8663,7 +8694,7 @@ indirectly through syntactic constructs
       <key>fstring-terminator-multi-tail</key>
       <dict>
         <key>begin</key>
-        <string>(![rsa])?(:)(?=.*?{)</string>
+        <string>((?:=?)(?:![rsa])?)(:)(?=.*?{)</string>
         <key>end</key>
         <string>(?=})</string>
         <key>beginCaptures</key>

diff --git a/grammars/src/MagicPython.syntax.yaml b/grammars/src/MagicPython.syntax.yaml
@@ -582,12 +582,15 @@ repository:
 
           | (!= | == | >= | <= | < | >)                          (?# 5)
 
+          | (:=)                                                 (?# 6)
+
     captures:
       '1': {name: keyword.operator.logical.python}
       '2': {name: keyword.control.flow.python}
       '3': {name: keyword.operator.bitwise.python}
       '4': {name: keyword.operator.arithmetic.python}
       '5': {name: keyword.operator.comparison.python}
+      '6': {name: keyword.operator.assignment.python}
 
   punctuation:
     patterns:
@@ -1002,6 +1005,8 @@ repository:
 
         contentName: meta.function.lambda.parameters.python
         patterns:
+          - name: keyword.operator.positional.parameter.python
+            match: /
           - name: keyword.operator.unpacking.parameter.python
             match: (\*\*|\*)
           - include: '#lambda-nested-incomplete'
@@ -1096,6 +1101,8 @@ repository:
       '1': {name: punctuation.definition.parameters.end.python}
 
     patterns:
+      - name: keyword.operator.positional.parameter.python
+        match: /
       - name: keyword.operator.unpacking.parameter.python
         match: (\*\*|\*)
       - include: '#lambda-incomplete'

diff --git a/grammars/src/pyfstring.inc.syntax.yaml b/grammars/src/pyfstring.inc.syntax.yaml
@@ -127,10 +127,12 @@ fstring-${line}-brace:
 fstring-terminator-${line}:
   patterns:
     - name: storage.type.format.python
-      match: (![rsa])(?=})
+      match: (=(![rsa])?)(?=})
+    - name: storage.type.format.python
+      match: (=?![rsa])(?=})
     - match: |
         (?x)
-          (![rsa])?
+          ( (?: =?) (?: ![rsa])? )
             ( : \w? [<>=^]? [-+ ]? \#?
               \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )(?=})
       captures:
@@ -148,7 +150,7 @@ fstring-terminator-${line}:
     - include: '#fstring-terminator-${line}-tail'
 
 fstring-terminator-${line}-tail:
-  begin: (![rsa])?(:)(?=.*?{)
+  begin: ((?:=?)(?:![rsa])?)(:)(?=.*?{)
   end: (?=})${fguard}
   beginCaptures:
     '1': {name: storage.type.format.python}

diff --git a/misc/example.py b/misc/example.py
@@ -1,7 +1,7 @@
 import asyncio
 
 
-def showcase():
+def showcase(a, b, /, c, d, *, e, f):
     """Some code to showcase the syntax.
 
     Docstrings are recognized and have an additional scope.
@@ -21,7 +21,7 @@ async def coroutine(db:aio_db.DatabaseConnection) -> List[str]:
 
         async with db.transaction():
             result = await db.query(...)
-            print(f'Result: {result!r}')
+            print(f'Result: {result!r} {a=} {b=!r}')
 
     mapping = None     # type: Dict[int, Any] # PEP 484
 
@@ -41,8 +41,14 @@ async def coroutine(db:aio_db.DatabaseConnection) -> List[str]:
     # NOTE Numbers with leading zeros are invalid in Python 3,
     # use 0o...
     answer = func(0xdeadbeef + 0b00100001 + 0123 + 0o123 +
-                  1_005_123 + # PEP 515
+                  1_005_123 +  # PEP 515
                   # complex numbers
                   .10e12 + 2j) @ mat
 
+    # walrus operator
+    filtered_data = [y for x in data if (y := f(x)) is not None]
+
+    # position-only params
+    bar = lambda q, w, /, e, r: (q + w + e + r)
+
     return R'''No escapes '\' in this \one'''
diff --git a/misc/scopes b/misc/scopes
@@ -56,6 +56,7 @@ keyword.operator.lookahead.regexp
 keyword.operator.lookbehind.negative.regexp
 keyword.operator.lookbehind.regexp
 keyword.operator.negation.regexp
+keyword.operator.positional.parameter.python
 keyword.operator.python
 keyword.operator.quantifier.regexp
 keyword.operator.unpacking.arguments.python