From e1f48aa47bf4acb36ae3292134d894e4d9e8f205 Mon Sep 17 00:00:00 2001 From: YannGarcia Date: Mon, 30 Sep 2019 05:25:14 -0700 Subject: [PATCH 01/30] Creare TTCN-3 lexer files --- lib/rouge/lexers/ttcn3.rb | 98 +++++++++++++++++++++++++++++++++++++++ spec/lexers/ttcn3_spec.rb | 20 ++++++++ 2 files changed, 118 insertions(+) create mode 100644 lib/rouge/lexers/ttcn3.rb create mode 100644 spec/lexers/ttcn3_spec.rb diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb new file mode 100644 index 0000000000..b3eeb2d004 --- /dev/null +++ b/lib/rouge/lexers/ttcn3.rb @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +module Rouge + module Lexers + module Ttcn3 < RegexLexer + title "Ttcn3" + desc "The Ttcn3 programming language (http://www.ttcn-3.org/). See ETSI ES 201 873-1" + + tag 'ttcn3' + filenames '*.ttcn', '*.ttcn3' + mimetypes 'text/x-ttcn3', 'text/x-ttcn' + + def self.keywords + @keywords ||= super + Set.new(%w( + module import group type port component signature external const template function altstep testcase var timer if else select case for while do label goto stop return break int2char int2unichar int2bit int2enum int2hex int2oct int2str int2float float2int char2int char2oct unichar2int unichar2oct bit2int bit2hex bit2oct bit2str hex2int hex2bit hex2oct hex2str oct2int oct2bit oct2hex oct2str oct2char oct2unichar str2int str2hex str2oct str2float enum2int any2unistr lengthof sizeof ispresent ischosen isvalue isbound istemplatekind regexp substr replace encvalue decvalue encvalue_unichar decvalue_unichar encvalue_o decvalue_o get_stringencoding remove_bom rnd hostid + )) + end + + def self.reserved + @reserved ||= super + Set.new(%w( + all apply assert at configuration conjunct const delta disjunct duration finished history implies inv mode notinv now omit onentry onexit par prev realtime seq setstate static stepsize stream timestamp until values wait + )) + end + + def self.types + @types ||= super + Set.new(%w( + anytype address boolean bitstring bytestring charstring component enumerated float integer hexstring octetstring port record set of union universal + )) + end + + id = /[a-zA-Z_][a-zA-Z0-9_]*/ + const_name = /[A-Z][A-Z0-9_]*\b/ + module_name = /[A-Z][a-zA-Z0-9]*\b/ + + state :root do + rule %r/[^\S\n]+/, Text + rule %r(//.*?$), Comment::Single + rule %r(/\*.*?\*/)m, Comment::Multiline + # keywords: go before method names to avoid lexing "throw new XYZ" + # as a method signature + rule %r/(?:#{keywords.join('|')})\b/, Keyword + + rule %r( + (\s*(?:[a-zA-Z_][a-zA-Z0-9_.\[\]<>]*\s+)+?) # return arguments + ([a-zA-Z_][a-zA-Z0-9_]*) # method name + (\s*)(\() # signature start + )mx do |m| + # TODO: do this better, this shouldn't need a delegation + delegate Ttcn3, m[1] + token Name::Function, m[2] + token Text, m[3] + token Operator, m[4] + end + + rule %r/@#{id}/, Name::Decorator + rule %r/(?:#{declarations.join('|')})\b/, Keyword::Declaration + rule %r/(?:#{types.join('|')})\b/, Keyword::Type + rule %r/package\b/, Keyword::Namespace + rule %r/(?:true|false|null)\b/, Keyword::Constant + rule %r/(?:module|interface)\b/, Keyword::Declaration, :module + rule %r/import\b/, Keyword::Namespace, :import + rule %r/"(\\\\|\\"|[^"])*"/, Str + rule %r/'(?:\\.|[^\\]|\\u[0-9a-f]{4})'/, Str::Char + rule %r/(\.)(#{id})/ do + groups Operator, Name::Attribute + end + + rule %r/#{id}:/, Name::Label + rule const_name, Name::Constant + rule module_name, Name::Module + rule %r/\$?#{id}/, Name + rule %r/[~^*!%&\[\](){}<>\|+=:;,.\/?-]/, Operator + + digit = /[0-9]_+[0-9]|[0-9]/ + bin_digit = /[01]_+[01]|[01]/ + oct_digit = /[0-7]_+[0-7]|[0-7]/ + hex_digit = /[0-9a-f]_+[0-9a-f]|[0-9a-f]/i + rule %r/#{digit}+\.#{digit}+([eE]#{digit}+)?[fd]?/, Num::Float + rule %r/0b#{bin_digit}+/i, Num::Bin + rule %r/0x#{hex_digit}+/i, Num::Hex + rule %r/0#{oct_digit}+/, Num::Oct + rule %r/#{digit}+L?/, Num::Integer + rule %r/\n/, Text + end + + state :module do + rule %r/\s+/m, Text + rule id, Name::Module, :pop! + end + + state :import do + rule %r/\s+/m, Text + rule %r/[a-z0-9_.]+\*?/i, Name::Namespace, :pop! + end + end + end +end diff --git a/spec/lexers/ttcn3_spec.rb b/spec/lexers/ttcn3_spec.rb new file mode 100644 index 0000000000..4422e424f5 --- /dev/null +++ b/spec/lexers/ttcn3_spec.rb @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +describe Rouge::Lexers::Ttcn3 do + let(:subject) { Rouge::Lexers::Ttcn3.new } + + describe 'guessing' do + include Support::Guessing + + it 'guesses by filename' do + assert_guess :filename => 'foo.ttcn' + assert_guess :filename => 'foo.ttcn3' + end + + it 'guesses by mimetype' do + assert_guess :mimetype => 'text/x-ttcn' + assert_guess :mimetype => 'text/x-ttcn3' + end + end +end From 968f9a34a22bcbafec952ed82b93ba3409d57064 Mon Sep 17 00:00:00 2001 From: YannGarcia Date: Mon, 30 Sep 2019 22:31:17 -0700 Subject: [PATCH 02/30] Add sample, Enhance TTCN-3 lexer --- lib/rouge/demos/ttcn3 | 120 ++++++++++++++++++++++++++++++++++++++ lib/rouge/lexers/ttcn3.rb | 40 ++++++------- 2 files changed, 137 insertions(+), 23 deletions(-) create mode 100644 lib/rouge/demos/ttcn3 diff --git a/lib/rouge/demos/ttcn3 b/lib/rouge/demos/ttcn3 new file mode 100644 index 0000000000..034926f0d9 --- /dev/null +++ b/lib/rouge/demos/ttcn3 @@ -0,0 +1,120 @@ +/* ---------------------------------------------------------------------------- + * (c) Copyright Wiley & Sons 2005 + * + * @author: Colin Willcock, Thomas Deiß, Stephan Tobies, Stefan Keil, Federico + * Engler, Stephan Schulz + * @desc: This is a strongly simplified Domain Name Server (DNS) test suite + * for testing some basic domain name resolution behaviour. + * @remark: This TTCN-3 code is based on the DNS example code presented in + * "C. Willock et al., An Introduction to TTCN-3, Wiley & Sons, 2005. + * ISBN: 0-470-01224-2" + * This copyright notice shall not be removed in copies of this file. + * ---------------------------------------------------------------------------- +*/ +module DNSTester { + + // Simple type definitions to match the protocol structure + type integer Identification( 0..65535 ); // 16-bit integer + type enumerated MessageKind {e_Question, e_Answer}; + type charstring Question; + type charstring Answer; + + // The definition of our DNS message type. + type record DNSMessage { + Identification identification, + MessageKind messageKind, + Question question, + Answer answer optional + } + + // A possible template for the DNS message type. + template DNSMessage a_NokiaQuestion := { + identification := 12345, + messageKind := e_Question, + question := "www.nokia.com", + answer := omit + } + + // A parameterized template for DNS questions based on DNSMessage. + template DNSMessage a_DNSQuestion( Identification p_id, Question p_question ) := { + identification := p_id, + messageKind := e_Question, + question := p_question, + answer := omit + } + + // A parameterized template for DNS answers based on DNSMessage. + template DNSMessage a_DNSAnswer( Identification p_id, Answer p_answer ) := { + identification := p_id, + messageKind := e_Answer, + question := ?, + answer := p_answer + } + + // DNS messages are allowed to move in and out through ports of this type. + type port DNSPort message { + inout DNSMessage + } + + // Our single component uses one single port to communicate with the SUT. + type component DNSClient { + port DNSPort serverPort + } + + // Our first test case! This small test case will behave very poorly in case + // of an erroneous SUT. More about this later! + testcase ExampleResolveNokia1() runs on DNSClient { + serverPort.send( a_DNSQuestion( 12345, "www.research.nokia.com" ) ); + serverPort.receive( a_DNSAnswer( 12345, "172.21.56.98" ) ); + setverdict( pass ); + stop; + } + + testcase ExampleResolveNokia2() runs on DNSClient { + serverPort.send( a_DNSQuestion( 12345, "www.research.nokia.com" ) ); + alt { + // Handle the case when the expected answer comes in. + [] serverPort.receive( a_DNSAnswer( 12345, "172.21.56.98" ) ) { + setverdict( pass ); + } + // Handle the case when unexpected answers come in. + [] serverPort.receive { + setverdict( fail ); + } + } + stop; + } + + // Our test case is now able to handle incorrect replies as well as + // missing replies. + testcase ExampleResolveNokia3() runs on DNSClient { + timer replyTimer; + serverPort.send( a_DNSQuestion( 12345, "www.research.nokia.com" ) ); + replyTimer.start( 20.0 ); + alt { + // Handle the case when the expected answer comes in. + [] serverPort.receive( a_DNSAnswer( 12345, "172.21.56.98" ) ) { + setverdict( pass ); + replyTimer.stop; + } + // Handle the case when unexpected answers come in. + [] serverPort.receive { + setverdict( fail ); + replyTimer.stop; + } + // Handle the case when no answer comes in. + [] replyTimer.timeout { + setverdict( fail ); + } + } + stop; + } + + // Our small control part. + control { + execute( ExampleResolveNokia1() ); + execute( ExampleResolveNokia2() ); + execute( ExampleResolveNokia3() ); + } + +} diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index b3eeb2d004..e74ad1a3aa 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -3,7 +3,7 @@ module Rouge module Lexers - module Ttcn3 < RegexLexer + class Ttcn3 < RegexLexer title "Ttcn3" desc "The Ttcn3 programming language (http://www.ttcn-3.org/). See ETSI ES 201 873-1" @@ -11,24 +11,20 @@ module Ttcn3 < RegexLexer filenames '*.ttcn', '*.ttcn3' mimetypes 'text/x-ttcn3', 'text/x-ttcn' - def self.keywords - @keywords ||= super + Set.new(%w( - module import group type port component signature external const template function altstep testcase var timer if else select case for while do label goto stop return break int2char int2unichar int2bit int2enum int2hex int2oct int2str int2float float2int char2int char2oct unichar2int unichar2oct bit2int bit2hex bit2oct bit2str hex2int hex2bit hex2oct hex2str oct2int oct2bit oct2hex oct2str oct2char oct2unichar str2int str2hex str2oct str2float enum2int any2unistr lengthof sizeof ispresent ischosen isvalue isbound istemplatekind regexp substr replace encvalue decvalue encvalue_unichar decvalue_unichar encvalue_o decvalue_o get_stringencoding remove_bom rnd hostid - )) - end + keywords = %w( + module import group type port component signature external execute const template function altstep testcase var timer if else select case for while do label goto start stop return break int2char int2unichar int2bit int2enum int2hex int2oct int2str int2float float2int char2int char2oct unichar2int unichar2oct bit2int bit2hex bit2oct bit2str hex2int hex2bit hex2oct hex2str oct2int oct2bit oct2hex oct2str oct2char oct2unichar str2int str2hex str2oct str2float enum2int any2unistr lengthof sizeof ispresent ischosen isvalue isbound istemplatekind regexp substr replace encvalue decvalue encvalue_unichar decvalue_unichar encvalue_o decvalue_o get_stringencoding remove_bom rnd hostid send receive setverdict + ) - def self.reserved - @reserved ||= super + Set.new(%w( - all apply assert at configuration conjunct const delta disjunct duration finished history implies inv mode notinv now omit onentry onexit par prev realtime seq setstate static stepsize stream timestamp until values wait - )) - end + reserved = %w( + all alt apply assert at configuration conjunct const control delta deterministic disjunct duration fail finished fuzzy history implies inconc inv lazy mod mode notinv now omit onentry onexit par pass prev realtime seq setstate static stepsize stream timestamp until values wait + ) - def self.types - @types ||= super + Set.new(%w( + types = %w( anytype address boolean bitstring bytestring charstring component enumerated float integer hexstring octetstring port record set of union universal - )) - end + ) + # optional comment or whitespace + ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+) id = /[a-zA-Z_][a-zA-Z0-9_]*/ const_name = /[A-Z][A-Z0-9_]*\b/ module_name = /[A-Z][a-zA-Z0-9]*\b/ @@ -40,7 +36,7 @@ def self.types # keywords: go before method names to avoid lexing "throw new XYZ" # as a method signature rule %r/(?:#{keywords.join('|')})\b/, Keyword - + rule %r{[~!@#\$%\^&\*\(\)\+`\-={}\[\]:;<>\?,\.\/\|\\]}, Punctuation rule %r( (\s*(?:[a-zA-Z_][a-zA-Z0-9_.\[\]<>]*\s+)+?) # return arguments ([a-zA-Z_][a-zA-Z0-9_]*) # method name @@ -54,11 +50,9 @@ def self.types end rule %r/@#{id}/, Name::Decorator - rule %r/(?:#{declarations.join('|')})\b/, Keyword::Declaration rule %r/(?:#{types.join('|')})\b/, Keyword::Type - rule %r/package\b/, Keyword::Namespace rule %r/(?:true|false|null)\b/, Keyword::Constant - rule %r/(?:module|interface)\b/, Keyword::Declaration, :module + rule %r/(module)\b/, Keyword::Declaration, :module rule %r/import\b/, Keyword::Namespace, :import rule %r/"(\\\\|\\"|[^"])*"/, Str rule %r/'(?:\\.|[^\\]|\\u[0-9a-f]{4})'/, Str::Char @@ -68,7 +62,7 @@ def self.types rule %r/#{id}:/, Name::Label rule const_name, Name::Constant - rule module_name, Name::Module + rule module_name, Name::Label rule %r/\$?#{id}/, Name rule %r/[~^*!%&\[\](){}<>\|+=:;,.\/?-]/, Operator @@ -77,9 +71,9 @@ def self.types oct_digit = /[0-7]_+[0-7]|[0-7]/ hex_digit = /[0-9a-f]_+[0-9a-f]|[0-9a-f]/i rule %r/#{digit}+\.#{digit}+([eE]#{digit}+)?[fd]?/, Num::Float - rule %r/0b#{bin_digit}+/i, Num::Bin - rule %r/0x#{hex_digit}+/i, Num::Hex - rule %r/0#{oct_digit}+/, Num::Oct + rule %r/'#{bin_digit}+'B/i, Num::Bin + rule %r/'#{hex_digit}+'H/i, Num::Hex + rule %r/'#{oct_digit}+'O/, Num::Oct rule %r/#{digit}+L?/, Num::Integer rule %r/\n/, Text end From 00c0da17ea3b424f638d5a490c0e73f6d4ef7fb9 Mon Sep 17 00:00:00 2001 From: YannGarcia Date: Tue, 1 Oct 2019 00:16:49 -0700 Subject: [PATCH 03/30] Add visual sample --- spec/visual/samples/ttcn3 | 120 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 spec/visual/samples/ttcn3 diff --git a/spec/visual/samples/ttcn3 b/spec/visual/samples/ttcn3 new file mode 100644 index 0000000000..034926f0d9 --- /dev/null +++ b/spec/visual/samples/ttcn3 @@ -0,0 +1,120 @@ +/* ---------------------------------------------------------------------------- + * (c) Copyright Wiley & Sons 2005 + * + * @author: Colin Willcock, Thomas Deiß, Stephan Tobies, Stefan Keil, Federico + * Engler, Stephan Schulz + * @desc: This is a strongly simplified Domain Name Server (DNS) test suite + * for testing some basic domain name resolution behaviour. + * @remark: This TTCN-3 code is based on the DNS example code presented in + * "C. Willock et al., An Introduction to TTCN-3, Wiley & Sons, 2005. + * ISBN: 0-470-01224-2" + * This copyright notice shall not be removed in copies of this file. + * ---------------------------------------------------------------------------- +*/ +module DNSTester { + + // Simple type definitions to match the protocol structure + type integer Identification( 0..65535 ); // 16-bit integer + type enumerated MessageKind {e_Question, e_Answer}; + type charstring Question; + type charstring Answer; + + // The definition of our DNS message type. + type record DNSMessage { + Identification identification, + MessageKind messageKind, + Question question, + Answer answer optional + } + + // A possible template for the DNS message type. + template DNSMessage a_NokiaQuestion := { + identification := 12345, + messageKind := e_Question, + question := "www.nokia.com", + answer := omit + } + + // A parameterized template for DNS questions based on DNSMessage. + template DNSMessage a_DNSQuestion( Identification p_id, Question p_question ) := { + identification := p_id, + messageKind := e_Question, + question := p_question, + answer := omit + } + + // A parameterized template for DNS answers based on DNSMessage. + template DNSMessage a_DNSAnswer( Identification p_id, Answer p_answer ) := { + identification := p_id, + messageKind := e_Answer, + question := ?, + answer := p_answer + } + + // DNS messages are allowed to move in and out through ports of this type. + type port DNSPort message { + inout DNSMessage + } + + // Our single component uses one single port to communicate with the SUT. + type component DNSClient { + port DNSPort serverPort + } + + // Our first test case! This small test case will behave very poorly in case + // of an erroneous SUT. More about this later! + testcase ExampleResolveNokia1() runs on DNSClient { + serverPort.send( a_DNSQuestion( 12345, "www.research.nokia.com" ) ); + serverPort.receive( a_DNSAnswer( 12345, "172.21.56.98" ) ); + setverdict( pass ); + stop; + } + + testcase ExampleResolveNokia2() runs on DNSClient { + serverPort.send( a_DNSQuestion( 12345, "www.research.nokia.com" ) ); + alt { + // Handle the case when the expected answer comes in. + [] serverPort.receive( a_DNSAnswer( 12345, "172.21.56.98" ) ) { + setverdict( pass ); + } + // Handle the case when unexpected answers come in. + [] serverPort.receive { + setverdict( fail ); + } + } + stop; + } + + // Our test case is now able to handle incorrect replies as well as + // missing replies. + testcase ExampleResolveNokia3() runs on DNSClient { + timer replyTimer; + serverPort.send( a_DNSQuestion( 12345, "www.research.nokia.com" ) ); + replyTimer.start( 20.0 ); + alt { + // Handle the case when the expected answer comes in. + [] serverPort.receive( a_DNSAnswer( 12345, "172.21.56.98" ) ) { + setverdict( pass ); + replyTimer.stop; + } + // Handle the case when unexpected answers come in. + [] serverPort.receive { + setverdict( fail ); + replyTimer.stop; + } + // Handle the case when no answer comes in. + [] replyTimer.timeout { + setverdict( fail ); + } + } + stop; + } + + // Our small control part. + control { + execute( ExampleResolveNokia1() ); + execute( ExampleResolveNokia2() ); + execute( ExampleResolveNokia3() ); + } + +} From 6ce8b5a5c0b4d0f7b17412aed32a0d95eee84462 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 7 Oct 2019 08:33:00 +0900 Subject: [PATCH 04/30] Reduce demo in lines of code --- lib/rouge/demos/ttcn3 | 114 ------------------------------------------ 1 file changed, 114 deletions(-) diff --git a/lib/rouge/demos/ttcn3 b/lib/rouge/demos/ttcn3 index 034926f0d9..969107de42 100644 --- a/lib/rouge/demos/ttcn3 +++ b/lib/rouge/demos/ttcn3 @@ -1,120 +1,6 @@ -/* ---------------------------------------------------------------------------- - * (c) Copyright Wiley & Sons 2005 - * - * @author: Colin Willcock, Thomas Deiß, Stephan Tobies, Stefan Keil, Federico - * Engler, Stephan Schulz - * @desc: This is a strongly simplified Domain Name Server (DNS) test suite - * for testing some basic domain name resolution behaviour. - * @remark: This TTCN-3 code is based on the DNS example code presented in - * "C. Willock et al., An Introduction to TTCN-3, Wiley & Sons, 2005. - * ISBN: 0-470-01224-2" - * This copyright notice shall not be removed in copies of this file. - * ---------------------------------------------------------------------------- -*/ module DNSTester { - - // Simple type definitions to match the protocol structure type integer Identification( 0..65535 ); // 16-bit integer type enumerated MessageKind {e_Question, e_Answer}; type charstring Question; type charstring Answer; - - // The definition of our DNS message type. - type record DNSMessage { - Identification identification, - MessageKind messageKind, - Question question, - Answer answer optional - } - - // A possible template for the DNS message type. - template DNSMessage a_NokiaQuestion := { - identification := 12345, - messageKind := e_Question, - question := "www.nokia.com", - answer := omit - } - - // A parameterized template for DNS questions based on DNSMessage. - template DNSMessage a_DNSQuestion( Identification p_id, Question p_question ) := { - identification := p_id, - messageKind := e_Question, - question := p_question, - answer := omit - } - - // A parameterized template for DNS answers based on DNSMessage. - template DNSMessage a_DNSAnswer( Identification p_id, Answer p_answer ) := { - identification := p_id, - messageKind := e_Answer, - question := ?, - answer := p_answer - } - - // DNS messages are allowed to move in and out through ports of this type. - type port DNSPort message { - inout DNSMessage - } - - // Our single component uses one single port to communicate with the SUT. - type component DNSClient { - port DNSPort serverPort - } - - // Our first test case! This small test case will behave very poorly in case - // of an erroneous SUT. More about this later! - testcase ExampleResolveNokia1() runs on DNSClient { - serverPort.send( a_DNSQuestion( 12345, "www.research.nokia.com" ) ); - serverPort.receive( a_DNSAnswer( 12345, "172.21.56.98" ) ); - setverdict( pass ); - stop; - } - - testcase ExampleResolveNokia2() runs on DNSClient { - serverPort.send( a_DNSQuestion( 12345, "www.research.nokia.com" ) ); - alt { - // Handle the case when the expected answer comes in. - [] serverPort.receive( a_DNSAnswer( 12345, "172.21.56.98" ) ) { - setverdict( pass ); - } - // Handle the case when unexpected answers come in. - [] serverPort.receive { - setverdict( fail ); - } - } - stop; - } - - // Our test case is now able to handle incorrect replies as well as - // missing replies. - testcase ExampleResolveNokia3() runs on DNSClient { - timer replyTimer; - serverPort.send( a_DNSQuestion( 12345, "www.research.nokia.com" ) ); - replyTimer.start( 20.0 ); - alt { - // Handle the case when the expected answer comes in. - [] serverPort.receive( a_DNSAnswer( 12345, "172.21.56.98" ) ) { - setverdict( pass ); - replyTimer.stop; - } - // Handle the case when unexpected answers come in. - [] serverPort.receive { - setverdict( fail ); - replyTimer.stop; - } - // Handle the case when no answer comes in. - [] replyTimer.timeout { - setverdict( fail ); - } - } - stop; - } - - // Our small control part. - control { - execute( ExampleResolveNokia1() ); - execute( ExampleResolveNokia2() ); - execute( ExampleResolveNokia3() ); - } - } From d646b4bfc5f2e4303dfe8e6572d98f9e73a42040 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 7 Oct 2019 08:41:19 +0900 Subject: [PATCH 05/30] Capitalise name of lexer --- lib/rouge/lexers/ttcn3.rb | 8 ++++---- spec/lexers/ttcn3_spec.rb | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index e74ad1a3aa..df12e9bc22 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -3,9 +3,9 @@ module Rouge module Lexers - class Ttcn3 < RegexLexer - title "Ttcn3" - desc "The Ttcn3 programming language (http://www.ttcn-3.org/). See ETSI ES 201 873-1" + class TTCN3 < RegexLexer + title "TTCN3" + desc "The TTCN3 programming language (http://www.ttcn-3.org/). See ETSI ES 201 873-1" tag 'ttcn3' filenames '*.ttcn', '*.ttcn3' @@ -43,7 +43,7 @@ module import group type port component signature external execute const templat (\s*)(\() # signature start )mx do |m| # TODO: do this better, this shouldn't need a delegation - delegate Ttcn3, m[1] + delegate TTCN3, m[1] token Name::Function, m[2] token Text, m[3] token Operator, m[4] diff --git a/spec/lexers/ttcn3_spec.rb b/spec/lexers/ttcn3_spec.rb index 4422e424f5..1acad980f1 100644 --- a/spec/lexers/ttcn3_spec.rb +++ b/spec/lexers/ttcn3_spec.rb @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # # frozen_string_literal: true -describe Rouge::Lexers::Ttcn3 do - let(:subject) { Rouge::Lexers::Ttcn3.new } +describe Rouge::Lexers::TTCN3 do + let(:subject) { Rouge::Lexers::TTCN3.new } describe 'guessing' do include Support::Guessing From e2f3a23992ea0c4799af4e143b851fc7d2f86058 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 7 Oct 2019 08:44:36 +0900 Subject: [PATCH 06/30] Shorten description --- lib/rouge/lexers/ttcn3.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index df12e9bc22..63f854626e 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -5,7 +5,7 @@ module Rouge module Lexers class TTCN3 < RegexLexer title "TTCN3" - desc "The TTCN3 programming language (http://www.ttcn-3.org/). See ETSI ES 201 873-1" + desc "The TTCN3 programming language (ttcn-3.org)" tag 'ttcn3' filenames '*.ttcn', '*.ttcn3' From 88df300c8dec58ce249b769c1bc2a198ddb5c9b7 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 7 Oct 2019 08:48:51 +0900 Subject: [PATCH 07/30] Wrap lists of defined words --- lib/rouge/lexers/ttcn3.rb | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index 63f854626e..85c19534b1 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -12,16 +12,33 @@ class TTCN3 < RegexLexer mimetypes 'text/x-ttcn3', 'text/x-ttcn' keywords = %w( - module import group type port component signature external execute const template function altstep testcase var timer if else select case for while do label goto start stop return break int2char int2unichar int2bit int2enum int2hex int2oct int2str int2float float2int char2int char2oct unichar2int unichar2oct bit2int bit2hex bit2oct bit2str hex2int hex2bit hex2oct hex2str oct2int oct2bit oct2hex oct2str oct2char oct2unichar str2int str2hex str2oct str2float enum2int any2unistr lengthof sizeof ispresent ischosen isvalue isbound istemplatekind regexp substr replace encvalue decvalue encvalue_unichar decvalue_unichar encvalue_o decvalue_o get_stringencoding remove_bom rnd hostid send receive setverdict - ) + module import group type port component signature external + execute const template function altstep testcase var timer if + else select case for while do label goto start stop return break + int2char int2unichar int2bit int2enum int2hex int2oct int2str + int2float float2int char2int char2oct unichar2int unichar2oct + bit2int bit2hex bit2oct bit2str hex2int hex2bit hex2oct hex2str + oct2int oct2bit oct2hex oct2str oct2char oct2unichar str2int + str2hex str2oct str2float enum2int any2unistr lengthof sizeof + ispresent ischosen isvalue isbound istemplatekind regexp substr + replace encvalue decvalue encvalue_unichar decvalue_unichar + encvalue_o decvalue_o get_stringencoding remove_bom rnd hostid + send receive setverdict + ) reserved = %w( - all alt apply assert at configuration conjunct const control delta deterministic disjunct duration fail finished fuzzy history implies inconc inv lazy mod mode notinv now omit onentry onexit par pass prev realtime seq setstate static stepsize stream timestamp until values wait - ) + all alt apply assert at configuration conjunct const control + delta deterministic disjunct duration fail finished fuzzy + history implies inconc inv lazy mod mode notinv now omit onentry + onexit par pass prev realtime seq setstate static stepsize stream + timestamp until values wait + ) types = %w( - anytype address boolean bitstring bytestring charstring component enumerated float integer hexstring octetstring port record set of union universal - ) + anytype address boolean bitstring bytestring charstring + component enumerated float integer hexstring octetstring port + record set of union universal + ) # optional comment or whitespace ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+) From 1ac95c6cc932823cd559d5bc85e99245acb6aae3 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 7 Oct 2019 08:52:10 +0900 Subject: [PATCH 08/30] Memoise defined words --- lib/rouge/lexers/ttcn3.rb | 59 ++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index 85c19534b1..e2a11fd3fe 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -11,34 +11,41 @@ class TTCN3 < RegexLexer filenames '*.ttcn', '*.ttcn3' mimetypes 'text/x-ttcn3', 'text/x-ttcn' - keywords = %w( - module import group type port component signature external - execute const template function altstep testcase var timer if - else select case for while do label goto start stop return break - int2char int2unichar int2bit int2enum int2hex int2oct int2str - int2float float2int char2int char2oct unichar2int unichar2oct - bit2int bit2hex bit2oct bit2str hex2int hex2bit hex2oct hex2str - oct2int oct2bit oct2hex oct2str oct2char oct2unichar str2int - str2hex str2oct str2float enum2int any2unistr lengthof sizeof - ispresent ischosen isvalue isbound istemplatekind regexp substr - replace encvalue decvalue encvalue_unichar decvalue_unichar - encvalue_o decvalue_o get_stringencoding remove_bom rnd hostid - send receive setverdict - ) + def self.keywords + @keywords = %w( + module import group type port component signature external + execute const template function altstep testcase var timer if + else select case for while do label goto start stop return + break int2char int2unichar int2bit int2enum int2hex int2oct + int2str int2float float2int char2int char2oct unichar2int + unichar2oct bit2int bit2hex bit2oct bit2str hex2int hex2bit + hex2oct hex2str oct2int oct2bit oct2hex oct2str oct2char + oct2unichar str2int str2hex str2oct str2float enum2int + any2unistr lengthof sizeof ispresent ischosen isvalue isbound + istemplatekind regexp substr replace encvalue decvalue + encvalue_unichar decvalue_unichar encvalue_o decvalue_o + get_stringencoding remove_bom rnd hostid send receive + setverdict + ) + end - reserved = %w( - all alt apply assert at configuration conjunct const control - delta deterministic disjunct duration fail finished fuzzy - history implies inconc inv lazy mod mode notinv now omit onentry - onexit par pass prev realtime seq setstate static stepsize stream - timestamp until values wait - ) + def self.reserved + @reserved = %w( + all alt apply assert at configuration conjunct const control + delta deterministic disjunct duration fail finished fuzzy + history implies inconc inv lazy mod mode notinv now omit + onentry onexit par pass prev realtime seq setstate static + stepsize stream timestamp until values wait + ) + end - types = %w( - anytype address boolean bitstring bytestring charstring - component enumerated float integer hexstring octetstring port - record set of union universal - ) + def self.types + @types = %w( + anytype address boolean bitstring bytestring charstring + component enumerated float integer hexstring octetstring port + record set of union universal + ) + end # optional comment or whitespace ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+) From cd2d02f7949c9a2f3b00d95d1569276d4bdb97ef Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 7 Oct 2019 18:15:41 +0900 Subject: [PATCH 09/30] Reorder rules to use memoised words --- lib/rouge/lexers/ttcn3.rb | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index e2a11fd3fe..0e81a6ccfa 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -57,9 +57,9 @@ def self.types rule %r/[^\S\n]+/, Text rule %r(//.*?$), Comment::Single rule %r(/\*.*?\*/)m, Comment::Multiline + # keywords: go before method names to avoid lexing "throw new XYZ" # as a method signature - rule %r/(?:#{keywords.join('|')})\b/, Keyword rule %r{[~!@#\$%\^&\*\(\)\+`\-={}\[\]:;<>\?,\.\/\|\\]}, Punctuation rule %r( (\s*(?:[a-zA-Z_][a-zA-Z0-9_.\[\]<>]*\s+)+?) # return arguments @@ -73,21 +73,34 @@ def self.types token Operator, m[4] end - rule %r/@#{id}/, Name::Decorator - rule %r/(?:#{types.join('|')})\b/, Keyword::Type rule %r/(?:true|false|null)\b/, Keyword::Constant rule %r/(module)\b/, Keyword::Declaration, :module rule %r/import\b/, Keyword::Namespace, :import - rule %r/"(\\\\|\\"|[^"])*"/, Str - rule %r/'(?:\\.|[^\\]|\\u[0-9a-f]{4})'/, Str::Char + rule const_name, Name::Constant + rule module_name, Name::Label + rule %r/(\.)(#{id})/ do groups Operator, Name::Attribute end - + rule %r/@#{id}/, Name::Decorator rule %r/#{id}:/, Name::Label - rule const_name, Name::Constant - rule module_name, Name::Label - rule %r/\$?#{id}/, Name + rule %r/\$#{id}/, Name + + rule id do |m| + if self.class.keywords.include? m[0] + token Keyword + elsif self.class.reserved.include? m[0] + # Was not used in submitted version + elsif self.class.types.include? m[0] + token Keyword::Type + else + token Name + end + end + + rule %r/"(\\\\|\\"|[^"])*"/, Str + rule %r/'(?:\\.|[^\\]|\\u[0-9a-f]{4})'/, Str::Char + rule %r/[~^*!%&\[\](){}<>\|+=:;,.\/?-]/, Operator digit = /[0-9]_+[0-9]|[0-9]/ @@ -104,7 +117,7 @@ def self.types state :module do rule %r/\s+/m, Text - rule id, Name::Module, :pop! + rule id, Name::Class, :pop! end state :import do From 26c29369f33a4cd8c13a969b399fe2b27bfe839c Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 7 Oct 2019 18:17:03 +0900 Subject: [PATCH 10/30] Add token for reserved words --- lib/rouge/lexers/ttcn3.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index 0e81a6ccfa..e58d509f39 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -90,7 +90,7 @@ def self.types if self.class.keywords.include? m[0] token Keyword elsif self.class.reserved.include? m[0] - # Was not used in submitted version + token Keyword::Reserved elsif self.class.types.include? m[0] token Keyword::Type else From 621906b840fff258846f5748b5c835bfa2b1ae64 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 7 Oct 2019 18:26:53 +0900 Subject: [PATCH 11/30] Rationalise operators and punctuation --- lib/rouge/lexers/ttcn3.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index e58d509f39..af639ce0a9 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -60,7 +60,6 @@ def self.types # keywords: go before method names to avoid lexing "throw new XYZ" # as a method signature - rule %r{[~!@#\$%\^&\*\(\)\+`\-={}\[\]:;<>\?,\.\/\|\\]}, Punctuation rule %r( (\s*(?:[a-zA-Z_][a-zA-Z0-9_.\[\]<>]*\s+)+?) # return arguments ([a-zA-Z_][a-zA-Z0-9_]*) # method name @@ -101,7 +100,8 @@ def self.types rule %r/"(\\\\|\\"|[^"])*"/, Str rule %r/'(?:\\.|[^\\]|\\u[0-9a-f]{4})'/, Str::Char - rule %r/[~^*!%&\[\](){}<>\|+=:;,.\/?-]/, Operator + rule %r/[~^*!%&<>\|+=\/?-]/, Operator + rule %r/[@#\$()`{}\[\]:;<>,.\\]/, Punctuation digit = /[0-9]_+[0-9]|[0-9]/ bin_digit = /[01]_+[01]|[01]/ From c11329f3f95eaeb225e0f92827cc8cf55af69b74 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 7 Oct 2019 18:28:12 +0900 Subject: [PATCH 12/30] Remove unnecessary newline rule --- lib/rouge/lexers/ttcn3.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index af639ce0a9..02df407e92 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -112,7 +112,6 @@ def self.types rule %r/'#{hex_digit}+'H/i, Num::Hex rule %r/'#{oct_digit}+'O/, Num::Oct rule %r/#{digit}+L?/, Num::Integer - rule %r/\n/, Text end state :module do From 55c7393457fe3365fa5335bb305c39c8ea28efea Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 7 Oct 2019 18:29:03 +0900 Subject: [PATCH 13/30] Remove duplicate characters from rules --- lib/rouge/lexers/ttcn3.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index 02df407e92..7e962e0020 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -101,7 +101,7 @@ def self.types rule %r/'(?:\\.|[^\\]|\\u[0-9a-f]{4})'/, Str::Char rule %r/[~^*!%&<>\|+=\/?-]/, Operator - rule %r/[@#\$()`{}\[\]:;<>,.\\]/, Punctuation + rule %r/[@#\$()`{}\[\]:;,.\\]/, Punctuation digit = /[0-9]_+[0-9]|[0-9]/ bin_digit = /[01]_+[01]|[01]/ From d199ed0c376f0b758491ac0e45b4894698166667 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 7 Oct 2019 18:30:25 +0900 Subject: [PATCH 14/30] Tidy up comment rules --- lib/rouge/lexers/ttcn3.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index 7e962e0020..01d9e97356 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -55,7 +55,8 @@ def self.types state :root do rule %r/[^\S\n]+/, Text - rule %r(//.*?$), Comment::Single + + rule %r(//.*), Comment::Single rule %r(/\*.*?\*/)m, Comment::Multiline # keywords: go before method names to avoid lexing "throw new XYZ" From 6dd49f62164741c407d6bb6d64fc1f2d3537d117 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Mon, 7 Oct 2019 18:40:54 +0900 Subject: [PATCH 15/30] Use numeric metacharacters --- lib/rouge/lexers/ttcn3.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index 01d9e97356..7f98dcbbf9 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -99,15 +99,15 @@ def self.types end rule %r/"(\\\\|\\"|[^"])*"/, Str - rule %r/'(?:\\.|[^\\]|\\u[0-9a-f]{4})'/, Str::Char + rule %r/'(?:\\.|[^\\]|\\u\h{4})'/, Str::Char rule %r/[~^*!%&<>\|+=\/?-]/, Operator rule %r/[@#\$()`{}\[\]:;,.\\]/, Punctuation - digit = /[0-9]_+[0-9]|[0-9]/ + digit = /\d_+\d|\d/ bin_digit = /[01]_+[01]|[01]/ oct_digit = /[0-7]_+[0-7]|[0-7]/ - hex_digit = /[0-9a-f]_+[0-9a-f]|[0-9a-f]/i + hex_digit = /\h_+\h|\h/ rule %r/#{digit}+\.#{digit}+([eE]#{digit}+)?[fd]?/, Num::Float rule %r/'#{bin_digit}+'B/i, Num::Bin rule %r/'#{hex_digit}+'H/i, Num::Hex From 6482586bd248e351f3a6421cefc1ab583ec89a1b Mon Sep 17 00:00:00 2001 From: YannGarcia Date: Mon, 14 Oct 2019 22:12:32 -0700 Subject: [PATCH 16/30] Add missing keywords & reserved identifiers --- lib/rouge/lexers/ttcn3.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index 7f98dcbbf9..fe79116346 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -32,7 +32,7 @@ module import group type port component signature external def self.reserved @reserved = %w( all alt apply assert at configuration conjunct const control - delta deterministic disjunct duration fail finished fuzzy + delta deterministic disjunct duration fail finished fuzzy from history implies inconc inv lazy mod mode notinv now omit onentry onexit par pass prev realtime seq setstate static stepsize stream timestamp until values wait @@ -41,17 +41,16 @@ def self.reserved def self.types @types = %w( - anytype address boolean bitstring bytestring charstring - component enumerated float integer hexstring octetstring port - record set of union universal + anytype address boolean bitstring charstring hexstring octetstring + component enumerated float integer port record set of union universal ) end # optional comment or whitespace ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+) id = /[a-zA-Z_][a-zA-Z0-9_]*/ - const_name = /[A-Z][A-Z0-9_]*\b/ - module_name = /[A-Z][a-zA-Z0-9]*\b/ + const_name = /[A-Z][a-zA-Z0-9_]*\b/ + module_name = /[A-Z][a-zA-Z0-9_]*\b/ state :root do rule %r/[^\S\n]+/, Text From b96f24309200acf127864993c9e19a93552430a5 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Tue, 15 Oct 2019 22:13:44 +0900 Subject: [PATCH 17/30] Add rule for newline --- lib/rouge/lexers/ttcn3.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index fe79116346..77326fb539 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -53,7 +53,7 @@ def self.types module_name = /[A-Z][a-zA-Z0-9_]*\b/ state :root do - rule %r/[^\S\n]+/, Text + rule %r/\s+/, Text rule %r(//.*), Comment::Single rule %r(/\*.*?\*/)m, Comment::Multiline From 369bae69d441275d8dcfcf01572699ff83de062c Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Tue, 15 Oct 2019 22:28:41 +0900 Subject: [PATCH 18/30] Properly memoise defined words --- lib/rouge/lexers/ttcn3.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index 77326fb539..b4c14a5643 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -12,7 +12,7 @@ class TTCN3 < RegexLexer mimetypes 'text/x-ttcn3', 'text/x-ttcn' def self.keywords - @keywords = %w( + @keywords ||= %w( module import group type port component signature external execute const template function altstep testcase var timer if else select case for while do label goto start stop return @@ -30,7 +30,7 @@ module import group type port component signature external end def self.reserved - @reserved = %w( + @reserved ||= %w( all alt apply assert at configuration conjunct const control delta deterministic disjunct duration fail finished fuzzy from history implies inconc inv lazy mod mode notinv now omit @@ -40,7 +40,7 @@ def self.reserved end def self.types - @types = %w( + @types ||= %w( anytype address boolean bitstring charstring hexstring octetstring component enumerated float integer port record set of union universal ) From 81c813e5805e3cbc7245e7a0013e71e5e14d8172 Mon Sep 17 00:00:00 2001 From: YannGarcia Date: Tue, 15 Oct 2019 07:09:03 -0700 Subject: [PATCH 19/30] Remove ws statement and useless comment --- lib/rouge/lexers/ttcn3.rb | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index b4c14a5643..7c9524ed11 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -46,24 +46,18 @@ def self.types ) end - # optional comment or whitespace - ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+) id = /[a-zA-Z_][a-zA-Z0-9_]*/ const_name = /[A-Z][a-zA-Z0-9_]*\b/ module_name = /[A-Z][a-zA-Z0-9_]*\b/ state :root do rule %r/\s+/, Text - rule %r(//.*), Comment::Single rule %r(/\*.*?\*/)m, Comment::Multiline - - # keywords: go before method names to avoid lexing "throw new XYZ" - # as a method signature rule %r( (\s*(?:[a-zA-Z_][a-zA-Z0-9_.\[\]<>]*\s+)+?) # return arguments - ([a-zA-Z_][a-zA-Z0-9_]*) # method name - (\s*)(\() # signature start + ([a-zA-Z_][a-zA-Z0-9_]*) # method name + (\s*)(\() # signature start )mx do |m| # TODO: do this better, this shouldn't need a delegation delegate TTCN3, m[1] From e7f9100526a2c2d7d04b9950980c4d05c1e5816e Mon Sep 17 00:00:00 2001 From: YannGarcia Date: Tue, 15 Oct 2019 07:20:45 -0700 Subject: [PATCH 20/30] Replace [a-zA-Z0-9_] by [\w] --- lib/rouge/lexers/ttcn3.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index 7c9524ed11..aa5e862bc4 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -46,18 +46,18 @@ def self.types ) end - id = /[a-zA-Z_][a-zA-Z0-9_]*/ - const_name = /[A-Z][a-zA-Z0-9_]*\b/ - module_name = /[A-Z][a-zA-Z0-9_]*\b/ + id = /[a-zA-Z_][\w]*/ + const_name = /[A-Z][\w]*\b/ + module_name = /[A-Z][\w]*\b/ state :root do rule %r/\s+/, Text rule %r(//.*), Comment::Single rule %r(/\*.*?\*/)m, Comment::Multiline rule %r( - (\s*(?:[a-zA-Z_][a-zA-Z0-9_.\[\]<>]*\s+)+?) # return arguments - ([a-zA-Z_][a-zA-Z0-9_]*) # method name - (\s*)(\() # signature start + (\s*(?:[a-zA-Z_][\w.\[\]<>]*\s+)+?) # return arguments + ([a-zA-Z_][\w]*) # method name + (\s*)(\() # signature start )mx do |m| # TODO: do this better, this shouldn't need a delegation delegate TTCN3, m[1] From d10a945309fa6893448e8848459630f570c48654 Mon Sep 17 00:00:00 2001 From: YannGarcia Date: Thu, 24 Oct 2019 23:33:08 -0700 Subject: [PATCH 21/30] Full review of TTCN-3 rules --- lib/rouge/lexers/ttcn3.rb | 135 +++++++++++++++++++++++++------------- 1 file changed, 89 insertions(+), 46 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index aa5e862bc4..e86d73d93b 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -46,66 +46,108 @@ def self.types ) end - id = /[a-zA-Z_][\w]*/ - const_name = /[A-Z][\w]*\b/ - module_name = /[A-Z][\w]*\b/ + def self.builtins + @builtins ||= [] + end - state :root do - rule %r/\s+/, Text - rule %r(//.*), Comment::Single - rule %r(/\*.*?\*/)m, Comment::Multiline - rule %r( - (\s*(?:[a-zA-Z_][\w.\[\]<>]*\s+)+?) # return arguments - ([a-zA-Z_][\w]*) # method name - (\s*)(\() # signature start - )mx do |m| - # TODO: do this better, this shouldn't need a delegation - delegate TTCN3, m[1] - token Name::Function, m[2] - token Text, m[3] - token Operator, m[4] - end + # optional comment or whitespace + ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+) + id = /[a-zA-Z_][a-zA-Z0-9_]*/ - rule %r/(?:true|false|null)\b/, Keyword::Constant - rule %r/(module)\b/, Keyword::Declaration, :module - rule %r/import\b/, Keyword::Namespace, :import - rule const_name, Name::Constant - rule module_name, Name::Label + state :inline_whitespace do + rule %r/[ \t\r]+/, Text + rule %r/\\\n/, Text # line continuation + rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline + end - rule %r/(\.)(#{id})/ do - groups Operator, Name::Attribute - end - rule %r/@#{id}/, Name::Decorator - rule %r/#{id}:/, Name::Label - rule %r/\$#{id}/, Name + state :whitespace do + rule %r/\n+/m, Text + rule %r(//(\\.|.)*?$), Comment::Single + mixin :inline_whitespace + end + state :expr_whitespace do + rule %r/\n+/m, Text + mixin :whitespace + end + + state :statements do + mixin :whitespace + digit = /\d_+\d|\d/ + bin_digit = /[01]_+[01]|[01]/ + oct_digit = /[0-7]_+[0-7]|[0-7]/ + hex_digit = /\h_+\h|\h/ + rule %r/"/, Str, :string + rule %r/'(?:\\.|[^\\]|\\u[0-9a-f]{4})'/, Str::Char + rule %r/#{digit}+\.#{digit}+([eE]#{digit}+)?[fd]?/i, Num::Float + rule %r/'#{bin_digit}+'B/i, Num::Bin + rule %r/'#{hex_digit}+'H/i, Num::Hex + rule %r/'#{oct_digit}+'O/i, Num::Oct + rule %r/#{digit}+L?/i, Num::Integer + rule %r(\*/), Error + rule %r([~!%^&*+:=\|?:<>/-]), Operator + rule %r/[()\[\],.;]/, Punctuation + rule %r/\bcase\b/, Keyword, :case + rule %r/(?:true|false|null)\b/, Name::Builtin rule id do |m| - if self.class.keywords.include? m[0] + name = m[0] + if self.class.keywords.include? name token Keyword - elsif self.class.reserved.include? m[0] - token Keyword::Reserved - elsif self.class.types.include? m[0] + elsif self.class.types.include? name token Keyword::Type + elsif self.class.reserved.include? name + token Keyword::Reserved + elsif self.class.builtins.include? name + token Name::Builtin else token Name end end + end + + state :case do + rule %r/:/, Punctuation, :pop! + mixin :statements + end + + state :root do + mixin :expr_whitespace + rule %r( + ([\w*\s]+?[\s*]) # return arguments + (#{id}) # function name + (\s*\([^;]*?\)) # signature + (#{ws}?)({|;) # open brace or semicolon + )mx do |m| + recurse m[1] + token Name::Function, m[2] + recurse m[3] + recurse m[4] + token Punctuation, m[5] + end - rule %r/"(\\\\|\\"|[^"])*"/, Str - rule %r/'(?:\\.|[^\\]|\\u\h{4})'/, Str::Char + rule %r/\{/, Punctuation, :function - rule %r/[~^*!%&<>\|+=\/?-]/, Operator - rule %r/[@#\$()`{}\[\]:;,.\\]/, Punctuation + rule %r/module\b/, Keyword::Declaration, :module - digit = /\d_+\d|\d/ - bin_digit = /[01]_+[01]|[01]/ - oct_digit = /[0-7]_+[0-7]|[0-7]/ - hex_digit = /\h_+\h|\h/ - rule %r/#{digit}+\.#{digit}+([eE]#{digit}+)?[fd]?/, Num::Float - rule %r/'#{bin_digit}+'B/i, Num::Bin - rule %r/'#{hex_digit}+'H/i, Num::Hex - rule %r/'#{oct_digit}+'O/, Num::Oct - rule %r/#{digit}+L?/, Num::Integer + rule %r/import\b/, Keyword::Namespace, :import + + mixin :statements + end + + state :function do + mixin :whitespace + mixin :statements + rule %r/;/, Punctuation + rule %r/{/, Punctuation, :function + rule %r/}/, Punctuation, :pop! + end + + state :string do + rule %r/"/, Str, :pop! + rule %r/\\([\\abfnrtv"']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})/, Str::Escape + rule %r/[^\\"\n]+/, Str + rule %r/\\\n/, Str + rule %r/\\/, Str # stray backslash end state :module do @@ -117,6 +159,7 @@ def self.types rule %r/\s+/m, Text rule %r/[a-z0-9_.]+\*?/i, Name::Namespace, :pop! end + end end end From 2f368935175aa8e37512dcd3f0eeb2e29b0a259a Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 27 Oct 2019 03:35:42 +0900 Subject: [PATCH 22/30] Use \w character class --- lib/rouge/lexers/ttcn3.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index e86d73d93b..03a09ffaf4 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -52,7 +52,7 @@ def self.builtins # optional comment or whitespace ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+) - id = /[a-zA-Z_][a-zA-Z0-9_]*/ + id = /[a-zA-Z_]\w*/ state :inline_whitespace do rule %r/[ \t\r]+/, Text @@ -157,7 +157,7 @@ def self.builtins state :import do rule %r/\s+/m, Text - rule %r/[a-z0-9_.]+\*?/i, Name::Namespace, :pop! + rule %r/[\w.]+\*?/, Name::Namespace, :pop! end end From 3a258dfd66184c1adf83ce063f6385e77f81a55a Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 27 Oct 2019 03:50:04 +0900 Subject: [PATCH 23/30] Remove unnecessary builtin variable --- lib/rouge/lexers/ttcn3.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index 03a09ffaf4..d4d3cc007a 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -46,10 +46,6 @@ def self.types ) end - def self.builtins - @builtins ||= [] - end - # optional comment or whitespace ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+) id = /[a-zA-Z_]\w*/ @@ -97,8 +93,6 @@ def self.builtins token Keyword::Type elsif self.class.reserved.include? name token Keyword::Reserved - elsif self.class.builtins.include? name - token Name::Builtin else token Name end From 105fddd37d307b8eef241d2795d0e600fc13ee7b Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 27 Oct 2019 03:54:02 +0900 Subject: [PATCH 24/30] Consolidate whitespace rules --- lib/rouge/lexers/ttcn3.rb | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index d4d3cc007a..fdaff56f2f 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -50,25 +50,14 @@ def self.types ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+) id = /[a-zA-Z_]\w*/ - state :inline_whitespace do + state :statements do + rule %r/\n+/m, Text rule %r/[ \t\r]+/, Text rule %r/\\\n/, Text # line continuation - rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline - end - state :whitespace do - rule %r/\n+/m, Text rule %r(//(\\.|.)*?$), Comment::Single - mixin :inline_whitespace - end - - state :expr_whitespace do - rule %r/\n+/m, Text - mixin :whitespace - end + rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline - state :statements do - mixin :whitespace digit = /\d_+\d|\d/ bin_digit = /[01]_+[01]|[01]/ oct_digit = /[0-7]_+[0-7]|[0-7]/ @@ -105,7 +94,6 @@ def self.types end state :root do - mixin :expr_whitespace rule %r( ([\w*\s]+?[\s*]) # return arguments (#{id}) # function name @@ -129,7 +117,6 @@ def self.types end state :function do - mixin :whitespace mixin :statements rule %r/;/, Punctuation rule %r/{/, Punctuation, :function From 0c7313e42bd2d33e4de85e0f77ebc04db2094731 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 27 Oct 2019 03:57:09 +0900 Subject: [PATCH 25/30] Insert spacing to improve visual dilenation --- lib/rouge/lexers/ttcn3.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index fdaff56f2f..ab9b167cbf 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -49,6 +49,10 @@ def self.types # optional comment or whitespace ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+) id = /[a-zA-Z_]\w*/ + digit = /\d_+\d|\d/ + bin_digit = /[01]_+[01]|[01]/ + oct_digit = /[0-7]_+[0-7]|[0-7]/ + hex_digit = /\h_+\h|\h/ state :statements do rule %r/\n+/m, Text @@ -58,22 +62,23 @@ def self.types rule %r(//(\\.|.)*?$), Comment::Single rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline - digit = /\d_+\d|\d/ - bin_digit = /[01]_+[01]|[01]/ - oct_digit = /[0-7]_+[0-7]|[0-7]/ - hex_digit = /\h_+\h|\h/ rule %r/"/, Str, :string rule %r/'(?:\\.|[^\\]|\\u[0-9a-f]{4})'/, Str::Char + rule %r/#{digit}+\.#{digit}+([eE]#{digit}+)?[fd]?/i, Num::Float rule %r/'#{bin_digit}+'B/i, Num::Bin rule %r/'#{hex_digit}+'H/i, Num::Hex rule %r/'#{oct_digit}+'O/i, Num::Oct rule %r/#{digit}+L?/i, Num::Integer + rule %r(\*/), Error + rule %r([~!%^&*+:=\|?:<>/-]), Operator rule %r/[()\[\],.;]/, Punctuation + rule %r/\bcase\b/, Keyword, :case rule %r/(?:true|false|null)\b/, Name::Builtin + rule id do |m| name = m[0] if self.class.keywords.include? name @@ -140,7 +145,6 @@ def self.types rule %r/\s+/m, Text rule %r/[\w.]+\*?/, Name::Namespace, :pop! end - end end end From c7930608dc8bc5b3065d0f7b1f1c25c49a696c23 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 27 Oct 2019 03:58:56 +0900 Subject: [PATCH 26/30] Remove function state --- lib/rouge/lexers/ttcn3.rb | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index ab9b167cbf..db9554ca4f 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -74,7 +74,7 @@ def self.types rule %r(\*/), Error rule %r([~!%^&*+:=\|?:<>/-]), Operator - rule %r/[()\[\],.;]/, Punctuation + rule %r/[()\[\]{},.;]/, Punctuation rule %r/\bcase\b/, Keyword, :case rule %r/(?:true|false|null)\b/, Name::Builtin @@ -112,8 +112,6 @@ def self.types token Punctuation, m[5] end - rule %r/\{/, Punctuation, :function - rule %r/module\b/, Keyword::Declaration, :module rule %r/import\b/, Keyword::Namespace, :import @@ -121,13 +119,6 @@ def self.types mixin :statements end - state :function do - mixin :statements - rule %r/;/, Punctuation - rule %r/{/, Punctuation, :function - rule %r/}/, Punctuation, :pop! - end - state :string do rule %r/"/, Str, :pop! rule %r/\\([\\abfnrtv"']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})/, Str::Escape From 4bd8c85f3254d952f9e2a60d97b3d037006a0a8a Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 27 Oct 2019 04:00:00 +0900 Subject: [PATCH 27/30] Remove function signature rule --- lib/rouge/lexers/ttcn3.rb | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index db9554ca4f..82cc562867 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -99,19 +99,6 @@ def self.types end state :root do - rule %r( - ([\w*\s]+?[\s*]) # return arguments - (#{id}) # function name - (\s*\([^;]*?\)) # signature - (#{ws}?)({|;) # open brace or semicolon - )mx do |m| - recurse m[1] - token Name::Function, m[2] - recurse m[3] - recurse m[4] - token Punctuation, m[5] - end - rule %r/module\b/, Keyword::Declaration, :module rule %r/import\b/, Keyword::Namespace, :import From c663f8d1c5a2f10cc9d084619fdaa4a41a2938b2 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 27 Oct 2019 04:00:25 +0900 Subject: [PATCH 28/30] Remove unnecessary line break --- lib/rouge/lexers/ttcn3.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index 82cc562867..1deace222c 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -100,7 +100,6 @@ def self.types state :root do rule %r/module\b/, Keyword::Declaration, :module - rule %r/import\b/, Keyword::Namespace, :import mixin :statements From 27a7c78c56f76a3a760be5693915d764e663c500 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sun, 27 Oct 2019 04:05:52 +0900 Subject: [PATCH 29/30] Remove case state --- lib/rouge/lexers/ttcn3.rb | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index 1deace222c..e8df053090 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -73,10 +73,9 @@ def self.types rule %r(\*/), Error - rule %r([~!%^&*+:=\|?:<>/-]), Operator - rule %r/[()\[\]{},.;]/, Punctuation + rule %r([~!%^&*+=\|?<>/-]), Operator + rule %r/[()\[\]{},.;:]/, Punctuation - rule %r/\bcase\b/, Keyword, :case rule %r/(?:true|false|null)\b/, Name::Builtin rule id do |m| @@ -93,11 +92,6 @@ def self.types end end - state :case do - rule %r/:/, Punctuation, :pop! - mixin :statements - end - state :root do rule %r/module\b/, Keyword::Declaration, :module rule %r/import\b/, Keyword::Namespace, :import From 58667e55f87aa2d38bb2835372002ed932b2a92a Mon Sep 17 00:00:00 2001 From: YannGarcia Date: Sun, 3 Nov 2019 03:15:25 -0800 Subject: [PATCH 30/30] Remove Error rule, enhance Integer rule --- lib/rouge/lexers/ttcn3.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/rouge/lexers/ttcn3.rb b/lib/rouge/lexers/ttcn3.rb index e8df053090..4b8b27d35c 100644 --- a/lib/rouge/lexers/ttcn3.rb +++ b/lib/rouge/lexers/ttcn3.rb @@ -69,11 +69,9 @@ def self.types rule %r/'#{bin_digit}+'B/i, Num::Bin rule %r/'#{hex_digit}+'H/i, Num::Hex rule %r/'#{oct_digit}+'O/i, Num::Oct - rule %r/#{digit}+L?/i, Num::Integer + rule %r/#{digit}+/i, Num::Integer - rule %r(\*/), Error - - rule %r([~!%^&*+=\|?<>/-]), Operator + rule %r([~!%^&*+:=\|?<>/-]), Operator rule %r/[()\[\]{},.;:]/, Punctuation rule %r/(?:true|false|null)\b/, Name::Builtin