From ed0726d9fb193fc1c0ef5b67892e5cee898d5edc Mon Sep 17 00:00:00 2001 From: Josh Pschorr Date: Fri, 10 Jun 2022 21:59:17 -0700 Subject: [PATCH] Add experimental REPL --- Cargo.toml | 1 + partiql-cli/Cargo.toml | 48 ++++ partiql-cli/LICENSE | 175 ++++++++++++++ partiql-cli/src/bin/ion.sublime-syntax | 142 +++++++++++ partiql-cli/src/bin/partiql-cli.rs | 260 +++++++++++++++++++++ partiql-cli/src/bin/partiql.sublime-syntax | 131 +++++++++++ partiql-parser/benches/bench_parse.rs | 6 +- partiql-parser/src/error.rs | 10 +- partiql-parser/src/lib.rs | 53 +++-- partiql-parser/src/parse/mod.rs | 64 ++--- partiql-parser/src/preprocessor.rs | 13 +- partiql-parser/src/token_parser.rs | 2 +- 12 files changed, 822 insertions(+), 83 deletions(-) create mode 100644 partiql-cli/Cargo.toml create mode 100644 partiql-cli/LICENSE create mode 100644 partiql-cli/src/bin/ion.sublime-syntax create mode 100644 partiql-cli/src/bin/partiql-cli.rs create mode 100644 partiql-cli/src/bin/partiql.sublime-syntax diff --git a/Cargo.toml b/Cargo.toml index aff5c4c7..fbddcea8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,4 +11,5 @@ members = [ "partiql-parser", "partiql-playground", "partiql-rewriter", + "partiql-cli" ] diff --git a/partiql-cli/Cargo.toml b/partiql-cli/Cargo.toml new file mode 100644 index 00000000..9dc9e695 --- /dev/null +++ b/partiql-cli/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "partiql-cli" +authors = ["PartiQL Team "] +description = "PartiQL CLI" +homepage = "https://github.com/partiql/partiql-lang-rust" +repository = "https://github.com/partiql/partiql-lang-rust" +license = "Apache-2.0" +readme = "../README.md" +keywords = ["sql", "parser", "query", "compilers", "interpreters"] +categories = ["database", "compilers", "parser-implementations"] +exclude = [ + "**/.git/**", + "**/.github/**", + "**/.travis.yml", + "**/.appveyor.yml", +] +edition = "2021" +version = "0.0.0" + +# Example of customizing binaries in Cargo.toml. +[[bin]] +name = "partiql-cli" +test = false +bench = false + + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustyline = "9.1.2" +syntect = "5.0" +owo-colors = "3.4.0" +supports-color = "1.3.0" +supports-unicode = "1.0.2" +supports-hyperlinks = "1.2.0" +termbg = "0.4.1" +shellexpand = "2.1.0" +partiql-parser = { path = "../partiql-parser" } +partiql-source-map = { path = "../partiql-source-map" } +partiql-ast = { path = "../partiql-ast" } + + +thiserror = "1.0.31" +miette = { version ="4.7.1", features = ["fancy"] } + + +tui = "0.18.0" +crossterm = "0.23.2" \ No newline at end of file diff --git a/partiql-cli/LICENSE b/partiql-cli/LICENSE new file mode 100644 index 00000000..67db8588 --- /dev/null +++ b/partiql-cli/LICENSE @@ -0,0 +1,175 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/partiql-cli/src/bin/ion.sublime-syntax b/partiql-cli/src/bin/ion.sublime-syntax new file mode 100644 index 00000000..5d82cbd6 --- /dev/null +++ b/partiql-cli/src/bin/ion.sublime-syntax @@ -0,0 +1,142 @@ +%YAML 1.2 +# See https://www.sublimetext.com/docs/syntax.htm +--- +name: ion +version: "2" +file_extensions: + - ion +scope: source.ion +contexts: + keywords: + - match: "\\b(?i:true|false)\\b" + scope: constant.language.bool.ion + - match: "\\b(?i:null.null|null.bool|null.int|null.float|null.decimal|null.timestamp|null.string|null.symbol|null.blob|null.clob|null.struct|null.list|null.sexp|null)\\b" + scope: constant.language.null.ion + main: + - include: value + value: + - include: whitespace + - include: comment + - include: annotation + - include: string + - include: number + - include: keywords + - include: symbol + - include: clob + - include: blob + - include: struct + - include: list + - include: sexp + sexp: + - match: "\\(" + scope: punctuation.definition.sexp.begin.ion + push: sexp__0 + sexp__0: + - match: "\\)" + scope: punctuation.definition.sexp.end.ion + pop: true + - include: comment + - include: value + - match: "[\\!\\#\\%\\&\\*\\+\\-\\.\\/\\;\\<\\=\\>\\?\\@\\^\\`\\|\\~]+" + scope: storage.type.symbol.operator.ion + comment: + - match: "\\/\\/[^\\n]*" + scope: comment.line.ion + - match: "\\/\\*" + scope: comment.block.ion + push: comment__1 + comment__1: + - match: "[*]\\/" + scope: comment.block.ion + pop: true + - match: "[^*\\/]+" + scope: comment.block.ion + - match: "[*\\/]+" + scope: comment.block.ion + list: + - match: "\\[" + scope: punctuation.definition.list.begin.ion + push: list__0 + list__0: + - match: "\\]" + scope: punctuation.definition.list.end.ion + pop: true + - include: comment + - include: value + - match: "," + scope: punctuation.definition.list.separator.ion + struct: + - match: "\\{" + scope: punctuation.definition.struct.begin.ion + push: struct__0 + struct__0: + - match: "\\}" + scope: punctuation.definition.struct.end.ion + pop: true + - include: comment + - include: value + - match: ",|:" + scope: punctuation.definition.struct.separator.ion + blob: + - match: "(\\{\\{)([^\"]*)(\\}\\})" + captures: + 1: punctuation.definition.blob.begin.ion + 2: string.other.blob.ion + 3: punctuation.definition.blob.end.ion + clob: + - match: "(\\{\\{)(\"[^\"]*\")(\\}\\})" + captures: + 1: punctuation.definition.clob.begin.ion + 2: string.other.clob.ion + 3: punctuation.definition.clob.end.ion + symbol: + - match: "(['])((?:(?:\\\\')|(?:[^']))*?)(['])" + scope: storage.type.symbol.quoted.ion + - match: "[\\$_a-zA-Z][\\$_a-zA-Z0-9]*" + scope: storage.type.symbol.identifier.ion + number: + - match: "\\d{4}(?:-\\d{2})?(?:-\\d{2})?T(?:\\d{2}:\\d{2})(?::\\d{2})?(?:\\.\\d+)?(?:Z|[-+]\\d{2}:\\d{2})?" + scope: constant.numeric.timestamp.ion + - match: "\\d{4}-\\d{2}-\\d{2}T?" + scope: constant.numeric.timestamp.ion + - match: "-?0[bB][01](?:_?[01])*" + scope: constant.numeric.integer.binary.ion + - match: "-?0[xX][0-9a-fA-F](?:_?[0-9a-fA-F])*" + scope: constant.numeric.integer.hex.ion + - match: "-?(?:0|[1-9](?:_?\\d)*)(?:\\.(?:\\d(?:_?\\d)*)?)?(?:[eE][+-]?\\d+)" + scope: constant.numeric.float.ion + - match: "(?:[-+]inf)|(?:nan)" + scope: constant.numeric.float.ion + - match: "-?(?:0|[1-9](?:_?\\d)*)(?:(?:(?:\\.(?:\\d(?:_?\\d)*)?)(?:[dD][+-]?\\d+)|\\.(?:\\d(?:_?\\d)*)?)|(?:[dD][+-]?\\d+))" + scope: constant.numeric.decimal.ion + - match: "-?(?:0|[1-9](?:_?\\d)*)" + scope: constant.numeric.integer.ion + string: + - match: "([\"])((?:(?:\\\\\")|(?:[^\"]))*?)([\"])" + captures: + 1: punctuation.definition.string.begin.ion + 2: string.quoted.double.ion + 3: punctuation.definition.string.end.ion + - match: "'{3}" + scope: punctuation.definition.string.begin.ion + push: string__1 + string__1: + - match: "'{3}" + scope: punctuation.definition.string.end.ion + pop: true + - match: "(?:\\\\'|[^'])+" + scope: string.quoted.triple.ion + - match: "'" + scope: string.quoted.triple.ion + annotation: + - match: "('(?:[^']|\\\\\\\\|\\\\')*')\\s*(::)" + captures: + 1: variable.language.annotation.ion + 2: punctuation.definition.annotation.ion + - match: "([\\$_a-zA-Z][\\$_a-zA-Z0-9]*)\\s*(::)" + captures: + 1: variable.language.annotation.ion + 2: punctuation.definition.annotation.ion + whitespace: + - match: "\\s+" + scope: text.ion \ No newline at end of file diff --git a/partiql-cli/src/bin/partiql-cli.rs b/partiql-cli/src/bin/partiql-cli.rs new file mode 100644 index 00000000..13084878 --- /dev/null +++ b/partiql-cli/src/bin/partiql-cli.rs @@ -0,0 +1,260 @@ +#![deny(rustdoc::broken_intra_doc_links)] + +use partiql_parser::ParseError; +use rustyline::completion::Completer; +use rustyline::config::Configurer; +use rustyline::highlight::Highlighter; +use rustyline::hint::{Hinter, HistoryHinter}; + +use rustyline::validate::{ValidationContext, ValidationResult, Validator}; +use rustyline::{ColorMode, Context, Helper}; +use std::borrow::Cow; +use std::fs::{File, OpenOptions}; + +use std::path::Path; + +use syntect::easy::HighlightLines; +use syntect::highlighting::{Style, ThemeSet}; +use syntect::parsing::{SyntaxDefinition, SyntaxSet, SyntaxSetBuilder}; +use syntect::util::as_24_bit_terminal_escaped; + +use miette::{Diagnostic, LabeledSpan, Report, SourceCode}; +use owo_colors::OwoColorize; +use partiql_source_map::location::{BytePosition, Location}; +use thiserror::Error; + +static ION_SYNTAX: &str = include_str!("ion.sublime-syntax"); +static PARTIQL_SYNTAX: &str = include_str!("partiql.sublime-syntax"); + +struct PartiqlHelperConfig { + dark_theme: bool, +} + +impl PartiqlHelperConfig { + pub fn infer() -> Self { + const TERM_TIMEOUT_MILLIS: u64 = 20; + let timeout = std::time::Duration::from_millis(TERM_TIMEOUT_MILLIS); + let theme = termbg::theme(timeout); + let dark_theme = match theme { + Ok(termbg::Theme::Light) => false, + Ok(termbg::Theme::Dark) => true, + _ => true, + }; + PartiqlHelperConfig { dark_theme } + } +} +struct PartiqlHelper { + config: PartiqlHelperConfig, + syntaxes: SyntaxSet, + themes: ThemeSet, +} + +impl PartiqlHelper { + pub fn new(config: PartiqlHelperConfig) -> Result { + let ion_def = SyntaxDefinition::load_from_str(ION_SYNTAX, false, Some("ion")).unwrap(); + let partiql_def = + SyntaxDefinition::load_from_str(PARTIQL_SYNTAX, false, Some("partiql")).unwrap(); + let mut builder = SyntaxSetBuilder::new(); + builder.add(ion_def); + builder.add(partiql_def); + + let syntaxes = builder.build(); + + let _ps = SyntaxSet::load_defaults_newlines(); + let themes = ThemeSet::load_defaults(); + Ok(PartiqlHelper { + config, + syntaxes, + themes, + }) + } +} + +impl Helper for PartiqlHelper {} + +impl Completer for PartiqlHelper { + type Candidate = String; +} +impl Hinter for PartiqlHelper { + type Hint = String; + + fn hint(&self, line: &str, pos: usize, ctx: &Context<'_>) -> Option { + let hinter = HistoryHinter {}; + hinter.hint(line, pos, ctx) + } +} +impl Highlighter for PartiqlHelper { + fn highlight<'l>(&self, line: &'l str, _pos: usize) -> Cow<'l, str> { + let syntax = self + .syntaxes + .find_syntax_by_extension("partiql") + .unwrap() + .clone(); + let theme = if self.config.dark_theme { + &self.themes.themes["Solarized (dark)"] + } else { + &self.themes.themes["Solarized (light)"] + }; + let mut highlighter = HighlightLines::new(&syntax, theme); + + let ranges: Vec<(Style, &str)> = highlighter.highlight_line(line, &self.syntaxes).unwrap(); + (as_24_bit_terminal_escaped(&ranges[..], true) + "\x1b[0m").into() + } + fn highlight_char(&self, line: &str, pos: usize) -> bool { + let _ = (line, pos); + true + } +} + +#[derive(Debug, Error)] +pub enum CLIError { + #[error("PartiQL syntax error:")] + SyntaxError { + src: String, + msg: String, + loc: Location, + }, + // TODO add github issue link + #[error("Internal Compiler Error - please report this.")] + InternalCompilerError { src: String }, +} + +impl Diagnostic for CLIError { + fn source_code(&self) -> Option<&dyn SourceCode> { + match self { + CLIError::SyntaxError { src, .. } => Some(src), + CLIError::InternalCompilerError { src, .. } => Some(src), + } + } + + fn labels(&self) -> Option + '_>> { + match self { + CLIError::SyntaxError { msg, loc, .. } => { + Some(Box::new(std::iter::once(LabeledSpan::new( + Some(msg.to_string()), + loc.start.0 .0 as usize, + loc.end.0 .0 as usize - loc.start.0 .0 as usize, + )))) + } + CLIError::InternalCompilerError { .. } => None, + } + } +} + +impl CLIError { + pub fn from_parser_error(err: ParseError, source: &str) -> CLIError { + match err { + ParseError::SyntaxError(partiql_source_map::location::Located { inner, location }) => { + CLIError::SyntaxError { + src: source.to_string(), + msg: format!("Syntax error `{}`", inner), + loc: location, + } + } + ParseError::UnexpectedToken(partiql_source_map::location::Located { + inner, + location, + }) => CLIError::SyntaxError { + src: source.to_string(), + msg: format!("Unexpected token `{}`", inner.token), + loc: location, + }, + ParseError::LexicalError(partiql_source_map::location::Located { inner, location }) => { + CLIError::SyntaxError { + src: source.to_string(), + msg: format!("Lexical error `{}`", inner), + loc: location, + } + } + ParseError::Unknown(location) => CLIError::SyntaxError { + src: source.to_string(), + msg: "Unknown parser error".to_string(), + loc: Location { + start: location, + end: location, + }, + }, + ParseError::IllegalState(_location) => CLIError::InternalCompilerError { + src: source.to_string(), + }, + _ => { + todo!("Not yet handled {:?}", err); + } + } + } +} + +impl Validator for PartiqlHelper { + fn validate(&self, ctx: &mut ValidationContext) -> rustyline::Result { + let parser = partiql_parser::Parser::default(); + let source = ctx.input(); + let result = parser.parse(source); + match result { + Ok(_) => Ok(ValidationResult::Valid(None)), + Err(e) => { + if e.errors + .iter() + .any(|err| matches!(err, ParseError::UnexpectedEndOfInput)) + { + // TODO For now, this is what allows you to do things like hit `` and continue writing the query on the next line in the middle of a query. + // TODO we should probably do something more ergonomic. Perhaps require a `;` or two newlines to end? + Ok(ValidationResult::Incomplete) + } else { + let err_msg = e + .errors + .into_iter() + .map(|e| CLIError::from_parser_error(e, source)) + .map(|e| format!("{:?}", Report::new(e))) + .collect::>() + .join("\n"); + Ok(ValidationResult::Invalid(Some(format!("\n\n{}", err_msg)))) + } + } + } + } +} + +fn main() -> miette::Result<()> { + let mut rl = rustyline::Editor::::new(); + rl.set_color_mode(ColorMode::Forced); + rl.set_helper(Some( + PartiqlHelper::new(PartiqlHelperConfig::infer()).unwrap(), + )); + let expanded = shellexpand::tilde("~/partiql_cli.history").to_string(); + let history_path = Path::new(&expanded); + OpenOptions::new() + .write(true) + .create(true) + .append(true) + .open(history_path) + .expect("history file create if not exists"); + rl.load_history(history_path).expect("history load"); + + println!("==============================="); + println!("PartiQL REPL"); + println!("CTRL-D on an empty line to quit"); + println!("==============================="); + + loop { + let readline = rl.readline(">> "); + match readline { + Ok(line) => { + println!("{}", "Parse OK!".green()); + rl.add_history_entry(line); + } + Err(_) => { + println!("Exiting..."); + rl.append_history(history_path).expect("append history"); + break; + } + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + #[test] + fn todo() {} +} diff --git a/partiql-cli/src/bin/partiql.sublime-syntax b/partiql-cli/src/bin/partiql.sublime-syntax new file mode 100644 index 00000000..acda916b --- /dev/null +++ b/partiql-cli/src/bin/partiql.sublime-syntax @@ -0,0 +1,131 @@ +%YAML 1.2 +# See https://www.sublimetext.com/docs/syntax.htm +--- +name: partiql +version: "2" +file_extensions: + - partiql +scope: source.partiql +contexts: + keywords: + - match: "\\b(?i:missing)\\b" + scope: constant.language.partiql + - match: "\\b(?i:false|null|true)\\b" + scope: constant.language.partiql + - match: "\\b(?i:pivot|unpivot|limit|tuple|remove|index|conflict|do|nothing|returning|modified|new|old|let)\\b" + scope: keyword.other.partiql + - match: "\\b(?i:absolute|action|add|all|allocate|alter|and|any|are|as|asc|assertion|at|authorization|begin|between|bit_length|by|cascade|cascaded|case|catalog|char|character_length|char_length|check|close|collate|collation|column|commit|connect|connection|constraint|constraints|continue|convert|corresponding|create|cross|current|cursor|deallocate|dec|declare|default|deferrable|deferred|delete|desc|describe|descriptor|diagnostics|disconnect|distinct|domain|drop|else|end|end-exec|escape|except|exception|exec|execute|external|extract|fetch|first|for|foreign|found|from|full|get|global|go|goto|grant|group|having|identity|immediate|in|indicator|initially|inner|input|insensitive|insert|intersect|interval|into|is|isolation|join|key|language|last|left|level|like|local|lower|match|module|names|national|natural|nchar|next|no|not|octet_length|of|on|only|open|option|or|order|outer|output|overlaps|pad|partial|position|precision|prepare|preserve|primary|prior|privileges|procedure|public|read|real|references|relative|restrict|revoke|right|rollback|rows|schema|scroll|section|select|session|set|size|some|space|sql|sqlcode|sqlerror|sqlstate|table|temporary|then|time|to|transaction|translate|translation|union|unique|unknown|update|upper|usage|user|using|value|values|view|when|whenever|where|with|work|write|zone)\\b" + scope: keyword.other.partiql + - match: "\\b(?i:bool|boolean|string|symbol|clob|blob|struct|list|sexp|bag)\\b" + scope: storage.type.partiql + - match: "\\b(?i:character|date|decimal|double|float|int|integer|numeric|smallint|timestamp|varchar|varying)\\b" + scope: storage.type.partiql + - match: "\\b(?i:avg|count|max|min|sum)\\b" + scope: support.function.aggregation.partiql + - match: "\\b(?i:cast|coalesce|current_date|current_time|current_timestamp|current_user|exists|date_add|date_diff|nullif|session_user|substring|system_user|trim)\\b" + scope: support.function.partiql + main: + - include: whitespace + - include: comment + - include: value + value: + - include: whitespace + - include: comment + - include: tuple_value + - include: collection_value + - include: scalar_value + scalar_value: + - include: string + - include: number + - include: keywords + - include: identifier + - match: "`" + captures: + 0: punctuation.definition.ion.begin.partiql + embed: "scope:source.ion" + escape: "`" + escape_captures: + 0: punctuation.definition.ion.end.partiql + - include: operator + - include: punctuation + punctuation: + - match: "[;:()\\[\\]\\{\\},.]" + scope: punctuation.partiql + operator: + - match: "[+*\\/<>=~!@#%&|?^-]+" + scope: keyword.operator.partiql + identifier: + - match: "([\"])((?:(?:\\\\.)|(?:[^\"\\\\]))*?)([\"])" + scope: variable.language.identifier.quoted.partiql + - match: "@\\w+" + scope: variable.language.identifier.at.partiql + - match: "\\b\\w+(?:\\.\\w+)?\\b" + scope: variable.language.identifier.partiql + number: + - match: "[+-]?\\d+(?:(?:\\.\\d*)?(?:[eE][+-]?\\d+)?)?\\b" + scope: constant.numeric.partiql + string: + - match: "(['])((?:(?:\\\\.)|(?:[^'\\\\]))*?)(['])" + captures: + 1: punctuation.definition.string.begin.partiql + 2: string.quoted.single.partiql + 3: punctuation.definition.string.end.partiql + collection_value: + - include: array_value + - include: bag_value + bag_value: + - match: "<<" + scope: punctuation.definition.bag.begin.partiql + push: bag_value__0 + bag_value__0: + - match: ">>" + scope: punctuation.definition.bag.end.partiql + pop: true + - include: comment + - match: "," + scope: punctuation.definition.bag.separator.partiql + - include: value + comment: + - match: "--.*" + scope: comment.line.partiql + - match: "\\/\\*" + scope: comment.block.partiql + push: comment__1 + comment__1: + - match: "[*]\\/" + scope: comment.block.partiql + pop: true + - match: "[^*\\/]+" + scope: comment.block.partiql + - match: "\\/\\*" + scope: comment.block.partiql + push: comment__1 + - match: "[*\\/]+" + scope: comment.block.partiql + array_value: + - match: "\\[" + scope: punctuation.definition.array.begin.partiql + push: array_value__0 + array_value__0: + - match: "\\]" + scope: punctuation.definition.array.end.partiql + pop: true + - include: comment + - match: "," + scope: punctuation.definition.array.separator.partiql + - include: value + tuple_value: + - match: "\\{" + scope: punctuation.definition.tuple.begin.partiql + push: tuple_value__0 + tuple_value__0: + - match: "\\}" + scope: punctuation.definition.tuple.end.partiql + pop: true + - include: comment + - match: ",|:" + scope: punctuation.definition.tuple.separator.partiql + - include: value + whitespace: + - match: "\\s+" + scope: text.partiql \ No newline at end of file diff --git a/partiql-parser/benches/bench_parse.rs b/partiql-parser/benches/bench_parse.rs index 319c8e7d..40a5daaf 100644 --- a/partiql-parser/benches/bench_parse.rs +++ b/partiql-parser/benches/bench_parse.rs @@ -1,5 +1,5 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use partiql_parser::parse_partiql; +use partiql_parser::{Parser, ParserResult}; use std::time::Duration; const Q_STAR: &str = "SELECT *"; @@ -35,7 +35,9 @@ const Q_COMPLEX_FEXPR: &str = r#" "#; fn parse_bench(c: &mut Criterion) { - let parse = parse_partiql; + fn parse(text: &str) -> ParserResult { + Parser::default().parse(text) + } c.bench_function("parse-simple", |b| b.iter(|| parse(black_box(Q_STAR)))); c.bench_function("parse-ion", |b| b.iter(|| parse(black_box(Q_ION)))); c.bench_function("parse-group", |b| b.iter(|| parse(black_box(Q_GROUP)))); diff --git a/partiql-parser/src/error.rs b/partiql-parser/src/error.rs index e7a11a55..1de585d6 100644 --- a/partiql-parser/src/error.rs +++ b/partiql-parser/src/error.rs @@ -5,17 +5,11 @@ use std::borrow::Cow; use std::fmt::{Debug, Display}; -use partiql_source_map::location::{LineAndColumn, Located}; +use partiql_source_map::location::Located; use thiserror::Error; use serde::{Deserialize, Serialize}; -/// [`Error`] type for errors in the lexical structure for the PartiQL parser. -pub type LexicalError<'input> = crate::error::LexError<'input>; - -/// [`Error`] type for errors in the syntactic structure for the PartiQL parser. -pub type ParserError<'input> = crate::error::ParseError<'input, LineAndColumn>; - /// Errors in the lexical structure of a PartiQL query. /// /// ### Notes @@ -57,7 +51,7 @@ where /// An otherwise un-categorized error occurred #[error("Unknown parse error at `{}`", _0)] - UnknownParseError(Loc), + Unknown(Loc), /// There was a token that was not expected #[error("Unexpected token `{}` at `{}`", _0.inner.token, _0.location)] diff --git a/partiql-parser/src/lib.rs b/partiql-parser/src/lib.rs index 7de56e7f..c2de0cf6 100644 --- a/partiql-parser/src/lib.rs +++ b/partiql-parser/src/lib.rs @@ -11,11 +11,11 @@ //! //! let parsed = parser.parse("SELECT g FROM data GROUP BY a").expect("successful parse"); //! -//! let errs: Vec = parser.parse("SELECT").expect_err("expected error"); +//! let errs: ParserError = parser.parse("SELECT").expect_err("expected error"); //! -//! let errs_at: Vec = +//! let errs_at: ParserError = //! parser.parse("SELECT * FROM a AY a CROSS JOIN c AS c AT q").unwrap_err(); -//! assert_eq!(errs_at[0].to_string(), "Unexpected token `` at `(1:20..1:21)`"); +//! assert_eq!(errs_at.errors[0].to_string(), "Unexpected token `` at `(b19..b20)`"); //! ``` //! //! [partiql]: https://partiql.org @@ -29,42 +29,55 @@ mod token_parser; use parse::parse_partiql; use partiql_ast::ast; use partiql_source_map::line_offset_tracker::LineOffsetTracker; +use partiql_source_map::location::BytePosition; -pub use error::LexError; -pub use error::LexicalError; -pub use error::ParseError; -pub use error::ParserError; +/// [`Error`] type for errors in the lexical structure for the PartiQL parser. +pub type LexicalError<'input> = error::LexError<'input>; + +/// [`Error`] type for errors in the syntactic structure for the PartiQL parser. +pub type ParseError<'input> = error::ParseError<'input, BytePosition>; use serde::{Deserialize, Serialize}; /// General [`Result`] type for the PartiQL [`Parser`]. -pub type ParserResult<'input> = Result, Vec>>; +pub type ParserResult<'input> = Result, ParserError<'input>>; /// A PartiQL parser from statement strings to AST. #[non_exhaustive] -#[derive(Debug)] +#[derive(Debug, Default)] pub struct Parser {} -impl Default for Parser { - fn default() -> Self { - Parser {} - } -} - impl Parser { /// Parse a PartiQL statement into an AST. pub fn parse<'input>(&self, text: &'input str) -> ParserResult<'input> { let mut offsets = LineOffsetTracker::default(); - let ast = parse_partiql(text, &mut offsets)?; - Ok(Parsed { text, offsets, ast }) + match parse_partiql(text, &mut offsets) { + Ok(ast) => Ok(Parsed { text, offsets, ast }), + Err(errors) => Err(ParserError { + text, + offsets, + errors, + }), + } } } /// The output of parsing PartiQL statement strings: an AST and auxiliary data. #[non_exhaustive] #[derive(Debug, Serialize, Deserialize)] +#[allow(dead_code)] pub struct Parsed<'input> { - text: &'input str, - offsets: LineOffsetTracker, - ast: Box, + pub text: &'input str, + pub offsets: LineOffsetTracker, + pub ast: Box, +} + +/// The output of errors when parsing PartiQL statement strings: an errors and auxiliary data. +#[non_exhaustive] +#[allow(dead_code)] +#[derive(Debug, Serialize, Deserialize)] +pub struct ParserError<'input> { + pub text: &'input str, + pub offsets: LineOffsetTracker, + pub errors: Vec>, } diff --git a/partiql-parser/src/parse/mod.rs b/partiql-parser/src/parse/mod.rs index 24942b04..10c237f5 100644 --- a/partiql-parser/src/parse/mod.rs +++ b/partiql-parser/src/parse/mod.rs @@ -3,13 +3,13 @@ //! Provides the [`parse_partiql`] function to parse a PartiQL query. use crate::error::{ParseError, UnexpectedTokenData}; +use crate::lexer; use crate::preprocessor::{built_ins, FnExprSet, PreprocessingPartiqlLexer}; -use crate::{lexer, ParserError}; use lalrpop_util as lpop; use lazy_static::lazy_static; use partiql_ast::ast; use partiql_source_map::line_offset_tracker::LineOffsetTracker; -use partiql_source_map::location::{ByteOffset, BytePosition, LineAndColumn, ToLocated}; +use partiql_source_map::location::{ByteOffset, BytePosition, ToLocated}; #[allow(clippy::just_underscores_and_digits)] // LALRPOP generates a lot of names like this #[allow(clippy::clone_on_copy)] @@ -30,7 +30,7 @@ type LalrpopResult<'input> = Result, LalrpopError<'input>>; type LalrpopErrorRecovery<'input> = lpop::ErrorRecovery, ParseError<'input, BytePosition>>; -pub(crate) type AstResult<'input> = Result, Vec>>; +pub(crate) type AstResult<'input> = Result, Vec>>; lazy_static! { static ref BUILT_INS: FnExprSet<'static> = built_ins(); @@ -46,32 +46,24 @@ pub fn parse_partiql<'input>(s: &'input str, offsets: &mut LineOffsetTracker) -> process_errors(s, offsets, parsed, errors) } -fn process_errors<'input, T>( - s: &'input str, - offsets: &LineOffsetTracker, - result: Result>, +fn process_errors<'input>( + _s: &'input str, + _offsets: &LineOffsetTracker, + result: Result, LalrpopError<'input>>, errors: Vec>, -) -> Result>> { - fn map_error<'input>( - s: &'input str, - offsets: &LineOffsetTracker, - e: LalrpopError<'input>, - ) -> ParseError<'input, LineAndColumn> { - ParseError::from(e).map_loc(|byte_loc| offsets.at(s, byte_loc).unwrap().into()) - } - +) -> AstResult<'input> { let mut parser_errors: Vec<_> = errors .into_iter() // TODO do something with error_recovery.dropped_tokens? - .map(|e| map_error(s, offsets, e.error)) + .map(|e| ParseError::from(e.error)) .collect(); match (result, parser_errors.is_empty()) { (Ok(ast), true) => Ok(ast), (Ok(_), false) => Err(parser_errors), - (Err(e), true) => Err(vec![map_error(s, offsets, e)]), + (Err(e), true) => Err(vec![ParseError::from(e)]), (Err(e), false) => { - parser_errors.push(map_error(s, offsets, e)); + parser_errors.push(ParseError::from(e)); Err(parser_errors) } } @@ -100,7 +92,7 @@ impl<'input> From> for ParseError<'input, BytePosition> { ), lalrpop_util::ParseError::InvalidToken { location } => { - ParseError::UnknownParseError(location.into()) + ParseError::Unknown(location.into()) } // TODO do something with UnrecognizedEOF.expected @@ -583,10 +575,8 @@ mod tests { mod errors { use super::*; - use crate::error::{LexicalError, UnexpectedToken, UnexpectedTokenData}; - use partiql_source_map::location::{ - CharOffset, LineAndCharPosition, LineOffset, Located, Location, - }; + use crate::error::{LexError, UnexpectedToken, UnexpectedTokenData}; + use partiql_source_map::location::{Located, Location}; use std::borrow::Cow; #[test] @@ -612,34 +602,18 @@ mod tests { token: Cow::from("/") }, location: Location { - start: LineAndCharPosition { - line: LineOffset(0), - char: CharOffset(0) - } - .into(), - end: LineAndCharPosition { - line: LineOffset(0), - char: CharOffset(1) - } - .into(), + start: BytePosition::from(0), + end: BytePosition::from(1), }, }) ); assert_eq!( errors[1], ParseError::LexicalError(Located { - inner: LexicalError::UnterminatedIonLiteral, + inner: LexError::UnterminatedIonLiteral, location: Location { - start: LineAndCharPosition { - line: LineOffset(0), - char: CharOffset(1) - } - .into(), - end: LineAndCharPosition { - line: LineOffset(0), - char: CharOffset(3) - } - .into(), + start: BytePosition::from(1), + end: BytePosition::from(4), }, }) ); diff --git a/partiql-parser/src/preprocessor.rs b/partiql-parser/src/preprocessor.rs index 37c53fbf..6b1586de 100644 --- a/partiql-parser/src/preprocessor.rs +++ b/partiql-parser/src/preprocessor.rs @@ -573,7 +573,6 @@ where mod tests { use super::*; use partiql_source_map::line_offset_tracker::LineOffsetTracker; - use partiql_source_map::location::BytePosition; use crate::ParseError; @@ -584,7 +583,7 @@ mod tests { } #[test] - fn cast() -> Result<(), ParseError<'static, BytePosition>> { + fn cast() -> Result<(), ParseError<'static>> { let query = "CAST(a AS VARCHAR)"; let mut offset_tracker = LineOffsetTracker::default(); @@ -609,7 +608,7 @@ mod tests { } #[test] - fn composed() -> Result<(), ParseError<'static, BytePosition>> { + fn composed() -> Result<(), ParseError<'static>> { let query = "cast(trim(LEADING 'Foo' from substring('BarFooBar' from 4 for 6)) AS VARCHAR(20))"; @@ -673,20 +672,20 @@ mod tests { } #[test] - fn preprocessor() -> Result<(), ParseError<'static, BytePosition>> { + fn preprocessor() -> Result<(), ParseError<'static>> { fn to_tokens<'a>( lexer: impl Iterator>, - ) -> Result>, ParseError<'a, BytePosition>> { + ) -> Result>, ParseError<'a>> { lexer .map(|result| result.map(|(_, t, _)| t)) .collect::, _>>() } - fn lex(query: &str) -> Result, ParseError> { + fn lex(query: &str) -> Result, ParseError> { let mut offset_tracker = LineOffsetTracker::default(); let lexer = PartiqlLexer::new(query, &mut offset_tracker); to_tokens(lexer) } - fn preprocess(query: &str) -> Result, ParseError> { + fn preprocess(query: &str) -> Result, ParseError> { let mut offset_tracker = LineOffsetTracker::default(); let lexer = PreprocessingPartiqlLexer::new(query, &mut offset_tracker, &*BUILT_INS); to_tokens(lexer) diff --git a/partiql-parser/src/token_parser.rs b/partiql-parser/src/token_parser.rs index 35271c5f..2f8b3cd6 100644 --- a/partiql-parser/src/token_parser.rs +++ b/partiql-parser/src/token_parser.rs @@ -1,5 +1,5 @@ +use crate::error::LexError; use crate::lexer::{PartiqlLexer, Spanned, Token}; -use crate::LexError; use partiql_source_map::location::ByteOffset; use std::collections::VecDeque;