From ee9865e17fe5e443a3361f627b5da4188c35adb2 Mon Sep 17 00:00:00 2001 From: Bastien Durel Date: Fri, 6 Mar 2020 10:18:04 +0100 Subject: [PATCH 1/8] Add Datastudio lexer --- lib/rouge/demos/datastudio | 21 ++++++ lib/rouge/lexers/datastudio.rb | 118 +++++++++++++++++++++++++++++++++ spec/lexers/datastudio.rb | 18 +++++ spec/visual/samples/datastudio | 51 ++++++++++++++ 4 files changed, 208 insertions(+) create mode 100644 lib/rouge/demos/datastudio create mode 100644 lib/rouge/lexers/datastudio.rb create mode 100644 spec/lexers/datastudio.rb create mode 100644 spec/visual/samples/datastudio diff --git a/lib/rouge/demos/datastudio b/lib/rouge/demos/datastudio new file mode 100644 index 0000000000..7c1de90533 --- /dev/null +++ b/lib/rouge/demos/datastudio @@ -0,0 +1,21 @@ + +Get_Variable("EUSER","ENV","USERNAME"); + +Message("Le Login Windows est : %EUSER%"); + +Get_Variable("st","JOB","FOLDER1.date_err.SYSTEM.STATUS"); + + +AFFECT("filter1", '%%/"t"'); +AFFECT("filter2", '%%/"pi"'); +JSONTOSQL("%{jsonpath}%/file.json", "", + " JSONPATH like '%filter1%' ", "a = JSONVALUE", + " JSONPATH like '%filter2%' ", "b = JSONVALUE; output(json_data, a, b)"); + + +Affect(VAR1,'%TEST%'); //Créer et affecter la variable VAR1 avec une chaîne +select * from TABLE1 where COL1 like %VAR1%; //utiliser la variable VAR1 dans une requête + +select * from TEST_TABLE; //exécution d'une requête Select pour ramener des valeurs +Affect_LastColumns("TEST1"); //création du paramètre TEST1 + diff --git a/lib/rouge/lexers/datastudio.rb b/lib/rouge/lexers/datastudio.rb new file mode 100644 index 0000000000..6e30704933 --- /dev/null +++ b/lib/rouge/lexers/datastudio.rb @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +module Rouge + module Lexers + class Datastudio < RegexLexer + tag 'datastudio' + filenames '*.job' + mimetypes 'text/x-datastudio' + + title "Datastudio" + desc 'Datastudio scripting language' + + id = /@?[_a-z]\w*/i + + sql_keywords = %w( + ABORT ABS ABSOLUTE ACCESS ADA ADD ADMIN AFTER AGGREGATE ALIAS + ALL ALLOCATE ALTER ANALYSE ANALYZE AND ANY ARE AS ASC ASENSITIVE + ASSERTION ASSIGNMENT ASYMMETRIC AT ATOMIC AUTHORIZATION + AVG BACKWARD BEFORE BEGIN BETWEEN BITVAR BIT_LENGTH BOTH + BREADTH BY C CACHE CALL CALLED CARDINALITY CASCADE CASCADED + CASE CAST CATALOG CATALOG_NAME CHAIN CHARACTERISTICS + CHARACTER_LENGTH CHARACTER_SET_CATALOG CHARACTER_SET_NAME + CHARACTER_SET_SCHEMA CHAR_LENGTH CHECK CHECKED CHECKPOINT + CLASS CLASS_ORIGIN CLOB CLOSE CLUSTER COALSECE COBOL COLLATE + COLLATION COLLATION_CATALOG COLLATION_NAME COLLATION_SCHEMA + COLUMN COLUMN_NAME COMMAND_FUNCTION COMMAND_FUNCTION_CODE + COMMENT COMMIT COMMITTED COMPLETION CONDITION_NUMBER + CONNECT CONNECTION CONNECTION_NAME CONSTRAINT CONSTRAINTS + CONSTRAINT_CATALOG CONSTRAINT_NAME CONSTRAINT_SCHEMA + CONSTRUCTOR CONTAINS CONTINUE CONVERSION CONVERT COPY + CORRESPONTING COUNT CREATE CREATEDB CREATEUSER CROSS CUBE + CURRENT CURRENT_DATE CURRENT_PATH CURRENT_ROLE CURRENT_TIME + CURRENT_TIMESTAMP CURRENT_USER CURSOR CURSOR_NAME CYCLE DATA + DATABASE DATETIME_INTERVAL_CODE DATETIME_INTERVAL_PRECISION + DAY DEALLOCATE DECLARE DEFAULT DEFAULTS DEFERRABLE DEFERRED + DEFINED DEFINER DELETE DELIMITER DELIMITERS DEREF DESC DESCRIBE + DESCRIPTOR DESTROY DESTRUCTOR DETERMINISTIC DIAGNOSTICS + DICTIONARY DISCONNECT DISPATCH DISTINCT DO DOMAIN DROP + DYNAMIC DYNAMIC_FUNCTION DYNAMIC_FUNCTION_CODE EACH ELSE + ENCODING ENCRYPTED END END-EXEC EQUALS ESCAPE EVERY EXCEPT + ESCEPTION EXCLUDING EXCLUSIVE EXEC EXECUTE EXISTING EXISTS + EXPLAIN EXTERNAL EXTRACT FALSE FETCH FINAL FIRST FOR FORCE + FOREIGN FORTRAN FORWARD FOUND FREE FREEZE FROM FULL FUNCTION + G GENERAL GENERATED GET GLOBAL GO GOTO GRANT GRANTED GROUP + GROUPING HANDLER HAVING HIERARCHY HOLD HOST IDENTITY IGNORE + ILIKE IMMEDIATE IMMUTABLE IMPLEMENTATION IMPLICIT IN INCLUDING + INCREMENT INDEX INDITCATOR INFIX INHERITS INITIALIZE INITIALLY + INNER INOUT INPUT INSENSITIVE INSERT INSTANTIABLE INSTEAD + INTERSECT INTO INVOKER IS ISNULL ISOLATION ITERATE JOIN KEY + KEY_MEMBER KEY_TYPE LANCOMPILER LANGUAGE LARGE LAST LATERAL + LEADING LEFT LENGTH LESS LEVEL LIKE LIMIT LISTEN LOAD LOCAL + LOCALTIME LOCALTIMESTAMP LOCATION LOCATOR LOCK LOWER MAP MATCH + MAX MAXVALUE MESSAGE_LENGTH MESSAGE_OCTET_LENGTH MESSAGE_TEXT + METHOD MIN MINUTE MINVALUE MOD MODE MODIFIES MODIFY MONTH + MORE MOVE MUMPS NAMES NATURAL NCLOB NEW NEXT + NO NOCREATEDB NOCREATEUSER NONE NOT NOTHING NOTIFY NOTNULL + NULL NULLABLE NULLIF OBJECT OCTET_LENGTH OF OFF OFFSET OIDS + OLD ON ONLY OPEN OPERATION OPERATOR OPTION OPTIONS OR ORDER + ORDINALITY OUT OUTER OUTPUT OVERLAPS OVERLAY OVERRIDING + OWNER PAD PARAMETER PARAMETERS PARAMETER_MODE PARAMATER_NAME + PARAMATER_ORDINAL_POSITION PARAMETER_SPECIFIC_CATALOG + PARAMETER_SPECIFIC_NAME PARAMATER_SPECIFIC_SCHEMA PARTIAL PASCAL + PENDANT PLACING PLI POSITION POSTFIX PREFIX PREORDER + PREPARE PRESERVE PRIMARY PRIOR PRIVILEGES PROCEDURAL PROCEDURE + PUBLIC READ READS RECHECK RECURSIVE REF REFERENCES REFERENCING + REINDEX RELATIVE RENAME REPEATABLE REPLACE RESET RESTART + RESTRICT RESULT RETURN RETURNED_LENGTH RETURNED_OCTET_LENGTH + RETURNED_SQLSTATE RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP + ROUTINE ROUTINE_CATALOG ROUTINE_NAME ROUTINE_SCHEMA ROW ROWS + ROW_COUNT RULE SAVE_POINT SCALE SCHEMA SCHEMA_NAME SCOPE SCROLL + SEARCH SECOND SECURITY SELECT SELF SENSITIVE SERIALIZABLE + SERVER_NAME SESSION SESSION_USER SET SETOF SETS SHARE SHOW + SIMILAR SIMPLE SIZE SOME SOURCE SPACE SPECIFIC SPECIFICTYPE + SPECIFIC_NAME SQL SQLCODE SQLERROR SQLEXCEPTION SQLSTATE + SQLWARNINIG STABLE START STATE STATEMENT STATIC STATISTICS + STDIN STDOUT STORAGE STRICT STRUCTURE STYPE SUBCLASS_ORIGIN + SUBLIST SUBSTRING SUM SYMMETRIC SYSID SYSTEM SYSTEM_USER + TABLE TABLE_NAME TEMP TEMPLATE TEMPORARY TERMINATE THAN THEN + TIMEZONE_HOUR TIMEZONE_MINUTE TO TOAST TRAILING + TRANSATION TRANSACTIONS_COMMITTED TRANSACTIONS_ROLLED_BACK + TRANSATION_ACTIVE TRANSFORM TRANSFORMS TRANSLATE TRANSLATION + TREAT TRIGGER TRIGGER_CATALOG TRIGGER_NAME TRIGGER_SCHEMA TRIM + TRUE TRUNCATE TRUSTED TYPE UNCOMMITTED UNDER UNENCRYPTED UNION + UNIQUE UNKNOWN UNLISTEN UNNAMED UNNEST UNTIL UPDATE UPPER + USAGE USER USER_DEFINED_TYPE_CATALOG USER_DEFINED_TYPE_NAME + USER_DEFINED_TYPE_SCHEMA USING VACUUM VALID VALIDATOR VALUES + VARIABLE VERBOSE VERSION VIEW VOLATILE WHEN WHENEVER WHERE + WITH WITHOUT WORK WRITE ZONE + ) + + state :whitespace do + rule %r/\s+/m, Text + rule %r(//.*?$), Comment::Single + rule %r(/[*].*?[*]/)m, Comment::Multiline + end + + state :root do + mixin :whitespace + + rule %r/^:#{id}/, Name::Label + rule %r/@#{id}(\.#{id})?/m, Name::Entity + rule %r/%(\\.|.)*?%/, Name::Variable + rule %r/[~!%^&*()+=|\[\]{}:;,.<>\/?-]/, Punctuation + rule %r/"(\\.|.|\n)*?"/, Str + rule %r/'(\\.|.|\n)*?'/, Str + rule %r( + [0-9] + ([.][0-9]*)? # decimal + )ix, Num + rule %r/#{id}(?=\s*[(])/, Name::Function + rule %r/\b(#{sql_keywords.join('|')})\b/i, Keyword + rule id, Name + end + + end + end +end diff --git a/spec/lexers/datastudio.rb b/spec/lexers/datastudio.rb new file mode 100644 index 0000000000..1aab65a84d --- /dev/null +++ b/spec/lexers/datastudio.rb @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +describe Rouge::Lexers::Datastudio do + let(:subject) { Rouge::Lexers::Datastudio.new } + + describe 'guessing' do + include Support::Guessing + + it 'guesses by filename' do + assert_guess :filename => 'foo.job' + end + + it 'guesses by mimetype' do + assert_guess :mimetype => 'text/x-datastudio' + end + end +end diff --git a/spec/visual/samples/datastudio b/spec/visual/samples/datastudio new file mode 100644 index 0000000000..faa6b9806e --- /dev/null +++ b/spec/visual/samples/datastudio @@ -0,0 +1,51 @@ + +Get_Variable("EUSER","ENV","USERNAME"); + +Message("Le Login Windows est : %EUSER%"); + +Get_Variable("st","JOB","FOLDER1.date_err.SYSTEM.STATUS"); + + +AFFECT("filter1", '%%/"t"'); +AFFECT("filter2", '%%/"pi"'); +JSONTOSQL("%{jsonpath}%/file.json", "", + " JSONPATH like '%filter1%' ", "a = JSONVALUE", + " JSONPATH like '%filter2%' ", "b = JSONVALUE; output(json_data, a, b)"); + +GOTO("There"); +/* +skipped +*/ +:There; + +Cursor("select * from IOUSER",,) +{ + Message("%USR%"); +} + +Affect(VAR1,'%TEST%'); //Créer et affecter la variable VAR1 avec une chaîne +select * from [TABLE1] where COL1 like %VAR1%; //utiliser la variable VAR1 dans une requête + +select * from @DATAZONE1.TEST_TABLE; //exécution d'une requête Select pour ramener des valeurs +Affect_LastColumns("TEST1", 0, 0); //création du paramètre TEST1 + +Affect("xslt", ' + + + + + + + : () + + + + + + +'); + +XSLT_MEM_SQL("out","select USRCODE, LOGONAT, NUMSESS from IOUSERS where rownum <= 2","%xslt%","CHARSET=ISO-8859-1"); + +Open_Sql("select * + from bar"); From 1f0366775f1e39cf7119d70214b2ad26d0b145ee Mon Sep 17 00:00:00 2001 From: Bastien Durel Date: Fri, 6 Mar 2020 12:29:02 +0100 Subject: [PATCH 2/8] allow nesting of variables in strings --- lib/rouge/lexers/datastudio.rb | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/rouge/lexers/datastudio.rb b/lib/rouge/lexers/datastudio.rb index 6e30704933..ef05953138 100644 --- a/lib/rouge/lexers/datastudio.rb +++ b/lib/rouge/lexers/datastudio.rb @@ -95,15 +95,27 @@ class Datastudio < RegexLexer rule %r(/[*].*?[*]/)m, Comment::Multiline end + state :string do + rule %r/%(\\.|.)+?%/, Str::Escape + rule %r/"/, Str, :pop! + rule %r/./, Str + end + + state :string_s do + rule %r/%(\\.|.)+?%/, Str::Escape + rule %r/'/, Str, :pop! + rule %r/./, Str + end + state :root do mixin :whitespace rule %r/^:#{id}/, Name::Label rule %r/@#{id}(\.#{id})?/m, Name::Entity - rule %r/%(\\.|.)*?%/, Name::Variable + rule %r/%(\\.|.)+?%/, Name::Variable rule %r/[~!%^&*()+=|\[\]{}:;,.<>\/?-]/, Punctuation - rule %r/"(\\.|.|\n)*?"/, Str - rule %r/'(\\.|.|\n)*?'/, Str + rule %r/"(\\.|.|\n)*?/, Str, :string + rule %r/'(\\.|.|\n)*?/, Str, :string_s rule %r( [0-9] ([.][0-9]*)? # decimal From 7fefd005574e872218b7c05df110c270a10b997c Mon Sep 17 00:00:00 2001 From: Bastien Durel Date: Fri, 6 Mar 2020 14:40:31 +0100 Subject: [PATCH 3/8] do not put errors in multi-line strings --- lib/rouge/lexers/datastudio.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/rouge/lexers/datastudio.rb b/lib/rouge/lexers/datastudio.rb index ef05953138..b893f715e2 100644 --- a/lib/rouge/lexers/datastudio.rb +++ b/lib/rouge/lexers/datastudio.rb @@ -97,14 +97,14 @@ class Datastudio < RegexLexer state :string do rule %r/%(\\.|.)+?%/, Str::Escape - rule %r/"/, Str, :pop! - rule %r/./, Str + rule %r/"/, Str::Double, :pop! + rule %r/.|\n/, Str::Double end state :string_s do rule %r/%(\\.|.)+?%/, Str::Escape - rule %r/'/, Str, :pop! - rule %r/./, Str + rule %r/'/, Str::Single, :pop! + rule %r/.|\n/, Str::Single end state :root do @@ -114,8 +114,8 @@ class Datastudio < RegexLexer rule %r/@#{id}(\.#{id})?/m, Name::Entity rule %r/%(\\.|.)+?%/, Name::Variable rule %r/[~!%^&*()+=|\[\]{}:;,.<>\/?-]/, Punctuation - rule %r/"(\\.|.|\n)*?/, Str, :string - rule %r/'(\\.|.|\n)*?/, Str, :string_s + rule %r/"/, Str::Double, :string + rule %r/'/, Str::Single, :string_s rule %r( [0-9] ([.][0-9]*)? # decimal From 58a9dd7cf71a8f0098f670984048fbf10ee1b028 Mon Sep 17 00:00:00 2001 From: Bastien Durel Date: Tue, 10 Mar 2020 10:54:49 +0100 Subject: [PATCH 4/8] Simplification of some regexes Avoid big local variable for sql keywords --- lib/rouge/lexers/datastudio.rb | 17 ++++++++--------- spec/visual/samples/datastudio | 4 ++++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/rouge/lexers/datastudio.rb b/lib/rouge/lexers/datastudio.rb index b893f715e2..de9bbd3399 100644 --- a/lib/rouge/lexers/datastudio.rb +++ b/lib/rouge/lexers/datastudio.rb @@ -13,7 +13,8 @@ class Datastudio < RegexLexer id = /@?[_a-z]\w*/i - sql_keywords = %w( + def self.sql_keywords + @sql_keywords ||= %w( ABORT ABS ABSOLUTE ACCESS ADA ADD ADMIN AFTER AGGREGATE ALIAS ALL ALLOCATE ALTER ANALYSE ANALYZE AND ANY ARE AS ASC ASENSITIVE ASSERTION ASSIGNMENT ASYMMETRIC AT ATOMIC AUTHORIZATION @@ -87,7 +88,8 @@ class Datastudio < RegexLexer USER_DEFINED_TYPE_SCHEMA USING VACUUM VALID VALIDATOR VALUES VARIABLE VERBOSE VERSION VIEW VOLATILE WHEN WHENEVER WHERE WITH WITHOUT WORK WRITE ZONE - ) + ) + end state :whitespace do rule %r/\s+/m, Text @@ -98,13 +100,13 @@ class Datastudio < RegexLexer state :string do rule %r/%(\\.|.)+?%/, Str::Escape rule %r/"/, Str::Double, :pop! - rule %r/.|\n/, Str::Double + rule %r/./m, Str::Double end state :string_s do rule %r/%(\\.|.)+?%/, Str::Escape rule %r/'/, Str::Single, :pop! - rule %r/.|\n/, Str::Single + rule %r/./m, Str::Single end state :root do @@ -116,12 +118,9 @@ class Datastudio < RegexLexer rule %r/[~!%^&*()+=|\[\]{}:;,.<>\/?-]/, Punctuation rule %r/"/, Str::Double, :string rule %r/'/, Str::Single, :string_s - rule %r( - [0-9] - ([.][0-9]*)? # decimal - )ix, Num + rule %r/\d(\.\d*)?/i, Num rule %r/#{id}(?=\s*[(])/, Name::Function - rule %r/\b(#{sql_keywords.join('|')})\b/i, Keyword + rule %r/\b(#{Datastudio.sql_keywords.join('|')})\b/i, Keyword rule id, Name end diff --git a/spec/visual/samples/datastudio b/spec/visual/samples/datastudio index faa6b9806e..b94dc7cf98 100644 --- a/spec/visual/samples/datastudio +++ b/spec/visual/samples/datastudio @@ -23,6 +23,10 @@ Cursor("select * from IOUSER",,) Message("%USR%"); } +/*** +test +***/ + Affect(VAR1,'%TEST%'); //Créer et affecter la variable VAR1 avec une chaîne select * from [TABLE1] where COL1 like %VAR1%; //utiliser la variable VAR1 dans une requête From 52f859d4b26d46f7bd9717caafc0e41af2ea7887 Mon Sep 17 00:00:00 2001 From: Bastien Durel Date: Tue, 10 Mar 2020 16:23:41 +0100 Subject: [PATCH 5/8] missing escaping of end of strings --- lib/rouge/lexers/datastudio.rb | 2 ++ spec/visual/samples/datastudio | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/lib/rouge/lexers/datastudio.rb b/lib/rouge/lexers/datastudio.rb index de9bbd3399..86b391bdea 100644 --- a/lib/rouge/lexers/datastudio.rb +++ b/lib/rouge/lexers/datastudio.rb @@ -99,12 +99,14 @@ def self.sql_keywords state :string do rule %r/%(\\.|.)+?%/, Str::Escape + rule %r/\\"/, Str::Double rule %r/"/, Str::Double, :pop! rule %r/./m, Str::Double end state :string_s do rule %r/%(\\.|.)+?%/, Str::Escape + rule %r/\\'/, Str::Single rule %r/'/, Str::Single, :pop! rule %r/./m, Str::Single end diff --git a/spec/visual/samples/datastudio b/spec/visual/samples/datastudio index b94dc7cf98..bff955e1ce 100644 --- a/spec/visual/samples/datastudio +++ b/spec/visual/samples/datastudio @@ -27,6 +27,27 @@ Cursor("select * from IOUSER",,) test ***/ +Affect('xsl',' + + + + + + + + + + + + + + +'); +Option("ESCAPE","False"); + +Rtf_Tab("C:\data\dev\rtf\rtf\fusion_tableaux.rtf","C:\data\dev\rtf\rtf\fusion_tableaux_res.rtf", + "TABLEAU1","select COL1,COL2,COL3 from TABLE_FUSIONRTF"); + Affect(VAR1,'%TEST%'); //Créer et affecter la variable VAR1 avec une chaîne select * from [TABLE1] where COL1 like %VAR1%; //utiliser la variable VAR1 dans une requête From 8e8cd1d5389e7f3417c5f93a077fe39fee59f4d4 Mon Sep 17 00:00:00 2001 From: Bastien Durel Date: Tue, 10 Mar 2020 16:24:49 +0100 Subject: [PATCH 6/8] missing escaping of end of strings --- spec/visual/samples/datastudio | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/visual/samples/datastudio b/spec/visual/samples/datastudio index bff955e1ce..f9af6e719c 100644 --- a/spec/visual/samples/datastudio +++ b/spec/visual/samples/datastudio @@ -27,6 +27,7 @@ Cursor("select * from IOUSER",,) test ***/ +Option("ESCAPE","True"); Affect('xsl',' From 9e37312b42306f27ebff938f7f8d606c34e6f03a Mon Sep 17 00:00:00 2001 From: Bastien Durel Date: Thu, 26 Mar 2020 10:01:28 +0100 Subject: [PATCH 7/8] use pyrmont's suggestion for improving performance Co-Authored-By: Michael Camilleri --- lib/rouge/lexers/datastudio.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/rouge/lexers/datastudio.rb b/lib/rouge/lexers/datastudio.rb index 86b391bdea..05b7352a05 100644 --- a/lib/rouge/lexers/datastudio.rb +++ b/lib/rouge/lexers/datastudio.rb @@ -123,7 +123,15 @@ def self.sql_keywords rule %r/\d(\.\d*)?/i, Num rule %r/#{id}(?=\s*[(])/, Name::Function rule %r/\b(#{Datastudio.sql_keywords.join('|')})\b/i, Keyword - rule id, Name + rule id do |m| + name = m[0].downcase + + if self.class.sql_keywords.include? name + token Keyword + else + token Name + end + end end end From 399d11de229315fbc3d8aa6b3e1ba3c71c518e62 Mon Sep 17 00:00:00 2001 From: Bastien Durel Date: Thu, 26 Mar 2020 10:17:07 +0100 Subject: [PATCH 8/8] match keywords agains upcase id, and remove big sql regex --- lib/rouge/lexers/datastudio.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/rouge/lexers/datastudio.rb b/lib/rouge/lexers/datastudio.rb index 05b7352a05..7508aaaa38 100644 --- a/lib/rouge/lexers/datastudio.rb +++ b/lib/rouge/lexers/datastudio.rb @@ -122,9 +122,8 @@ def self.sql_keywords rule %r/'/, Str::Single, :string_s rule %r/\d(\.\d*)?/i, Num rule %r/#{id}(?=\s*[(])/, Name::Function - rule %r/\b(#{Datastudio.sql_keywords.join('|')})\b/i, Keyword rule id do |m| - name = m[0].downcase + name = m[0].upcase if self.class.sql_keywords.include? name token Keyword