Add YANG lexer (#348)

* add yang lexer. See #347 * Alphabetise all keywords in yang lexer
alecthomas · Apr 14, 2020 · 28041a8 · 28041a8
1 parent 737b3a0
commit 28041a8
Show file tree

Hide file tree

Showing 3 changed files with 351 additions and 0 deletions.
diff --git a/lexers/testdata/yang.actual b/lexers/testdata/yang.actual
@@ -0,0 +1,64 @@
+module server-system {
+    yang-version 1.1;
+    namespace "http://autlan.dt/gribok/yang/example";
+    prefix ex;
+
+    import ietf-yang-types {
+      prefix yang;
+      reference
+        "RFC 6991: Common YANG Data Types.";
+    }
+
+    organization "Gribok";
+    contact "gribok@example.org";
+    description
+        "An example module";
+
+    revision 2020-04-03 {
+        description "Example yang";
+    }
+
+    /*
+     * Comment for container system
+     */
+
+    container system {
+        leaf host-name {
+            type string;
+            description "Hostname for this system";
+        }
+
+        leaf-list domain-search {
+            type string;
+            description "List of domain names to search";
+        }
+
+        container login {
+            leaf message {
+                type string;
+                description
+                    "Message given at start of login session";
+            }
+
+            list user {
+                key "name";
+                leaf name {
+                    type string;
+                }
+                leaf uuid {
+                  type yang:uuid;
+                }
+                leaf full-name {
+                    type string;
+                    mandatory true;
+                    description
+                      "The full name of user  See also 'name'.  This could
+                       be, for example, a reference to the user name";
+                }
+                leaf class {
+                    type string;
+                }
+            }
+        }
+    }
+}
diff --git a/lexers/testdata/yang.expected b/lexers/testdata/yang.expected
@@ -0,0 +1,220 @@
+[
+  {"type":"Keyword","value":"module"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"server-system"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"Keyword","value":"yang-version"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralNumberFloat","value":"1.1"},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"Keyword","value":"namespace"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralStringDouble","value":"\"http://autlan.dt/gribok/yang/example\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"Keyword","value":"prefix"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"ex"},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n\n    "},
+  {"type":"Keyword","value":"import"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"ietf-yang-types"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n      "},
+  {"type":"Keyword","value":"prefix"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"yang"},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n      "},
+  {"type":"Keyword","value":"reference"},
+  {"type":"TextWhitespace","value":"\n        "},
+  {"type":"LiteralStringDouble","value":"\"RFC 6991: Common YANG Data Types.\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n\n    "},
+  {"type":"Keyword","value":"organization"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralStringDouble","value":"\"Gribok\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"Keyword","value":"contact"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralStringDouble","value":"\"gribok@example.org\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"Keyword","value":"description"},
+  {"type":"TextWhitespace","value":"\n        "},
+  {"type":"LiteralStringDouble","value":"\"An example module\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n\n    "},
+  {"type":"Keyword","value":"revision"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralDate","value":"2020-04-03"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n        "},
+  {"type":"Keyword","value":"description"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralStringDouble","value":"\"Example yang\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n\n    "},
+  {"type":"CommentMultiline","value":"/*\n     * Comment for container system\n     */"},
+  {"type":"TextWhitespace","value":"\n\n    "},
+  {"type":"Keyword","value":"container"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"system"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n        "},
+  {"type":"Keyword","value":"leaf"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"host-name"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n            "},
+  {"type":"Keyword","value":"type"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameClass","value":"string"},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n            "},
+  {"type":"Keyword","value":"description"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralStringDouble","value":"\"Hostname for this system\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n        "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n\n        "},
+  {"type":"Keyword","value":"leaf-list"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"domain-search"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n            "},
+  {"type":"Keyword","value":"type"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameClass","value":"string"},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n            "},
+  {"type":"Keyword","value":"description"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralStringDouble","value":"\"List of domain names to search\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n        "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n\n        "},
+  {"type":"Keyword","value":"container"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"login"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n            "},
+  {"type":"Keyword","value":"leaf"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"message"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n                "},
+  {"type":"Keyword","value":"type"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameClass","value":"string"},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n                "},
+  {"type":"Keyword","value":"description"},
+  {"type":"TextWhitespace","value":"\n                    "},
+  {"type":"LiteralStringDouble","value":"\"Message given at start of login session\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n            "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n\n            "},
+  {"type":"Keyword","value":"list"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameClass","value":"user"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n                "},
+  {"type":"Keyword","value":"key"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"LiteralStringDouble","value":"\"name\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n                "},
+  {"type":"Keyword","value":"leaf"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"name"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n                    "},
+  {"type":"Keyword","value":"type"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameClass","value":"string"},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n                "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n                "},
+  {"type":"Keyword","value":"leaf"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"uuid"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n                  "},
+  {"type":"Keyword","value":"type"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"KeywordNamespace","value":"yang"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":"uuid"},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n                "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n                "},
+  {"type":"Keyword","value":"leaf"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"full-name"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n                    "},
+  {"type":"Keyword","value":"type"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameClass","value":"string"},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n                    "},
+  {"type":"Keyword","value":"mandatory"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameClass","value":"true"},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n                    "},
+  {"type":"Keyword","value":"description"},
+  {"type":"TextWhitespace","value":"\n                      "},
+  {"type":"LiteralStringDouble","value":"\"The full name of user  See also 'name'.  This could\n                       be, for example, a reference to the user name\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n                "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n                "},
+  {"type":"Keyword","value":"leaf"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Text","value":"class"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n                    "},
+  {"type":"Keyword","value":"type"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"NameClass","value":"string"},
+  {"type":"Punctuation","value":";"},
+  {"type":"TextWhitespace","value":"\n                "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n            "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n        "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n"}
+]
diff --git a/lexers/y/yang.go b/lexers/y/yang.go
@@ -0,0 +1,67 @@
+package y
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+var YANG = internal.Register(MustNewLexer(
+	&Config{
+		Name:      "YANG",
+		Aliases:   []string{"yang"},
+		Filenames: []string{"*.yang"},
+		MimeTypes: []string{"application/yang"},
+	},
+	Rules{
+		"root": {
+			{`\s+`, Whitespace, nil},
+			{`[\{\}\;]+`, Punctuation, nil},
+			{`(?<![\-\w])(and|or|not|\+|\.)(?![\-\w])`, Operator, nil},
+
+			{`"(?:\\"|[^"])*?"`, StringDouble, nil},
+			{`'(?:\\'|[^'])*?'`, StringSingle, nil},
+
+			{`/\*`, CommentMultiline, Push("comments")},
+			{`//.*?$`, CommentSingle, nil},
+
+			//match BNF stmt for `node-identifier` with [ prefix ":"]
+			{`(?:^|(?<=[\s{};]))([\w.-]+)(:)([\w.-]+)(?=[\s{};])`, ByGroups(KeywordNamespace, Punctuation, Text), nil},
+
+			//match BNF stmt `date-arg-str`
+			{`([0-9]{4}\-[0-9]{2}\-[0-9]{2})(?=[\s\{\}\;])`, LiteralDate, nil},
+			{`([0-9]+\.[0-9]+)(?=[\s\{\}\;])`, NumberFloat, nil},
+			{`([0-9]+)(?=[\s\{\}\;])`, NumberInteger, nil},
+
+			//TOP_STMTS_KEYWORDS
+			{Words(``, `(?=[^\w\-\:])`, `module`, `submodule`), Keyword, nil},
+			//MODULE_HEADER_STMT_KEYWORDS
+			{Words(``, `(?=[^\w\-\:])`, `belongs-to`, `namespace`, `prefix`, `yang-version`), Keyword, nil},
+			//META_STMT_KEYWORDS
+			{Words(``, `(?=[^\w\-\:])`, `contact`, `description`, `organization`, `reference`, `revision`), Keyword, nil},
+			//LINKAGE_STMTS_KEYWORDS
+			{Words(``, `(?=[^\w\-\:])`, `import`, `include`, `revision-date`), Keyword, nil},
+			//BODY_STMT_KEYWORDS
+			{Words(``, `(?=[^\w\-\:])`, `action`, `argument`, `augment`, `deviation`, `extension`, `feature`, `grouping`, `identity`, `if-feature`, `input`, `notification`, `output`, `rpc`, `typedef`), Keyword, nil},
+			//DATA_DEF_STMT_KEYWORDS
+			{Words(``, `(?=[^\w\-\:])`, `anydata`, `anyxml`, `case`, `choice`, `config`, `container`, `deviate`, `leaf`, `leaf-list`, `list`, `must`, `presence`, `refine`, `uses`, `when`), Keyword, nil},
+			//TYPE_STMT_KEYWORDS
+			{Words(``, `(?=[^\w\-\:])`, `base`, `bit`, `default`, `enum`, `error-app-tag`, `error-message`, `fraction-digits`, `length`, `max-elements`, `min-elements`, `modifier`, `ordered-by`, `path`, `pattern`, `position`, `range`, `require-instance`, `status`, `type`, `units`, `value`, `yin-element`), Keyword, nil},
+			//LIST_STMT_KEYWORDS
+			{Words(``, `(?=[^\w\-\:])`, `key`, `mandatory`, `unique`), Keyword, nil},
+
+			//CONSTANTS_KEYWORDS - RFC7950 other keywords
+			{Words(``, `(?=[^\w\-\:])`, `add`, `current`, `delete`, `deprecated`, `false`, `invert-match`, `max`, `min`, `not-supported`, `obsolete`, `replace`, `true`, `unbounded`, `user`), NameClass, nil},
+
+			//RFC7950 Built-In Types
+			{Words(``, `(?=[^\w\-\:])`, `binary`, `bits`, `boolean`, `decimal64`, `empty`, `enumeration`, `identityref`, `instance-identifier`, `int16`, `int32`, `int64`, `int8`, `leafref`, `string`, `uint16`, `uint32`, `uint64`, `uint8`, `union`), NameClass, nil},
+
+			{`[^;{}\s\'\"]+`, Text, nil},
+		},
+		"comments": {
+			{`[^*/]`, CommentMultiline, nil},
+			{`/\*`, CommentMultiline, Push("comment")},
+			{`\*/`, CommentMultiline, Pop(1)},
+			{`[*/]`, CommentMultiline, nil},
+		},
+	},
+))