Merge pull request #4401 from samstrohkorbatt/samstrohkorbatt/python-…

…fstring Adding Python f-string syntax support
microsoft · May 17, 2024 · a845ff6 · a845ff6
2 parents d919f28 + c869a18
commit a845ff6
Show file tree

Hide file tree

Showing 2 changed files with 88 additions and 0 deletions.
diff --git a/src/basic-languages/python/python.test.ts b/src/basic-languages/python/python.test.ts
@@ -189,5 +189,69 @@ testTokenization('python', [
 			line: '456.7e-7j',
 			tokens: [{ startIndex: 0, type: 'number.python' }]
 		}
+	],
+
+	// F-Strings
+	[
+		{
+			line: 'f"str {var} str"',
+			tokens: [
+				{ startIndex: 0, type: 'string.escape.python' },
+				{ startIndex: 2, type: 'string.python' },
+				{ startIndex: 6, type: 'identifier.python' },
+				{ startIndex: 11, type: 'string.python' },
+				{ startIndex: 15, type: 'string.escape.python' }
+			]
+		}
+	],
+	[
+		{
+			line: `f'''str {var} str'''`,
+			tokens: [
+				{ startIndex: 0, type: 'string.escape.python' },
+				{ startIndex: 4, type: 'string.python' },
+				{ startIndex: 8, type: 'identifier.python' },
+				{ startIndex: 13, type: 'string.python' },
+				{ startIndex: 17, type: 'string.escape.python' }
+			]
+		}
+	],
+	[
+		{
+			line: 'f"{var:.3f}{var!r}{var=}"',
+			tokens: [
+				{ startIndex: 0, type: 'string.escape.python' },
+				{ startIndex: 2, type: 'identifier.python' },
+				{ startIndex: 6, type: 'string.python' },
+				{ startIndex: 10, type: 'identifier.python' },
+				{ startIndex: 15, type: 'string.python' },
+				{ startIndex: 17, type: 'identifier.python' },
+				{ startIndex: 22, type: 'string.python' },
+				{ startIndex: 23, type: 'identifier.python' },
+				{ startIndex: 24, type: 'string.escape.python' }
+			]
+		}
+	],
+	[
+		{
+			line: 'f"\' " "',
+			tokens: [
+				{ startIndex: 0, type: 'string.escape.python' },
+				{ startIndex: 2, type: 'string.python' },
+				{ startIndex: 4, type: 'string.escape.python' },
+				{ startIndex: 5, type: 'white.python' },
+				{ startIndex: 6, type: 'string.escape.python' }
+			]
+		}
+	],
+	[
+		{
+			line: '"{var}"',
+			tokens: [
+				{ startIndex: 0, type: 'string.escape.python' },
+				{ startIndex: 1, type: 'string.python' },
+				{ startIndex: 6, type: 'string.escape.python' }
+			]
+		}
 	]
 ]);
diff --git a/src/basic-languages/python/python.ts b/src/basic-languages/python/python.ts
@@ -250,23 +250,47 @@ export const language = <languages.IMonarchLanguage>{
 		// Recognize strings, including those broken across lines with \ (but not without)
 		strings: [
 			[/'$/, 'string.escape', '@popall'],
+			[/f'{1,3}/, 'string.escape', '@fStringBody'],
 			[/'/, 'string.escape', '@stringBody'],
 			[/"$/, 'string.escape', '@popall'],
+			[/f"{1,3}/, 'string.escape', '@fDblStringBody'],
 			[/"/, 'string.escape', '@dblStringBody']
 		],
+		fStringBody: [
+			[/[^\\'\{\}]+$/, 'string', '@popall'],
+			[/[^\\'\{\}]+/, 'string'],
+			[/\{[^\}':!=]+/, 'identifier', '@fStringDetail'],
+			[/\\./, 'string'],
+			[/'/, 'string.escape', '@popall'],
+			[/\\$/, 'string']
+		],
 		stringBody: [
 			[/[^\\']+$/, 'string', '@popall'],
 			[/[^\\']+/, 'string'],
 			[/\\./, 'string'],
 			[/'/, 'string.escape', '@popall'],
 			[/\\$/, 'string']
 		],
+		fDblStringBody: [
+			[/[^\\"\{\}]+$/, 'string', '@popall'],
+			[/[^\\"\{\}]+/, 'string'],
+			[/\{[^\}':!=]+/, 'identifier', '@fStringDetail'],
+			[/\\./, 'string'],
+			[/"/, 'string.escape', '@popall'],
+			[/\\$/, 'string']
+		],
 		dblStringBody: [
 			[/[^\\"]+$/, 'string', '@popall'],
 			[/[^\\"]+/, 'string'],
 			[/\\./, 'string'],
 			[/"/, 'string.escape', '@popall'],
 			[/\\$/, 'string']
+		],
+		fStringDetail: [
+			[/[:][^}]+/, 'string'],
+			[/[!][ars]/, 'string'], // only !a, !r, !s are supported by f-strings: https://docs.python.org/3/tutorial/inputoutput.html#formatted-string-literals
+			[/=/, 'string'],
+			[/\}/, 'identifier', '@pop']
 		]
 	}
 };