add a Varnish (VCL) lexer (#365)

lib/rouge/demos/varnish

@@ -0,0 +1,55 @@
+vcl 4.0;
+
+backend server1 {
+      .host = "server1.example.com";
+      .probe = {
+              .url = "/";
+              .timeout = 1s;
+              .interval = 5s;
+              .window = 5;
+              .threshold = 3;
+      }
+}
+
+sub vcl_hit {
+    # Called when a cache lookup is successful.
+
+    if (obj.ttl >= 0s) {
+        # A pure unadultered hit, deliver it
+        return (deliver);
+    }
+
+    # https://www.varnish-cache.org/docs/trunk/users-guide/vcl-grace.html
+    # When several clients are requesting the same page Varnish will send one request to the backend and place the others on hold while fetching one copy from the backend. In some products this is called request coalescing and Varnish does this automatically.
+    # If you are serving thousands of hits per second the queue of waiting requests can get huge. There are two potential problems - one is a thundering herd problem - suddenly releasing a thousand threads to serve content might send the load sky high. Secondly - nobody likes to wait. To deal with this we can instruct Varnish to keep the objects in cache beyond their TTL and to serve the waiting requests somewhat stale content.
+
+# if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) {
+#     return (deliver);
+# } else {
+#     return (fetch);
+# }
+
+    # We have no fresh fish. Lets look at the stale ones.
+    if (std.healthy(req.backend_hint)) {
+        # Backend is healthy. Limit age to 10s.
+        if (obj.ttl + 10s > 0s) {
+            #set req.http.grace = "normal(limited)";
+            return (deliver);
+        } else {
+            # No candidate for grace. Fetch a fresh object.
+            return(fetch);
+        }
+    } else {
+        # backend is sick - use full grace
+            if (obj.ttl + obj.grace > 0s) {
+            #set req.http.grace = "full";
+            return (deliver);
+        } else {
+            # no graced object.
+            return (fetch);
+        }
+    }
+
+    # fetch & deliver once we get the result
+    return (fetch); # Dead code, keep as a safeguard
+}

lib/rouge/lexers/varnish.rb

@@ -0,0 +1,224 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+    module Lexers
+        class Varnish < RegexLexer
+            aliases 'varnishconf', 'VCL'
+            title 'Varnish'
+            desc 'The Varnish (high-performance web accelerator) configuration language'
+            tag 'varnish'
+            filenames '*.vcl'
+            mimetypes 'text/x-varnish'
+
+            LNUM = '[0-9]+'
+            DNUM = '([0-9]*"."[0-9]+)|([0-9]+"."[0-9]*)'
+            SPACE = '[ \f\n\r\t\v]+'
+#             IDENT1 = [a-zA-Z]
+#             IDENT = IDENT1 | [0-9_-]
+#             VAR = IDENT | '.'
+
+            # backend acl
+            KEYWORDS = Set.new %w[vcl set unset include import if else elseif elif elsif]
+
+            BUILTIN_FUNCTIONS = Set.new %w[
+                ban
+                call
+                hash_data
+                new
+                regsub
+                regsuball
+                return
+                rollback
+                std.cache_req_body
+                std.collect
+                std.duration
+                std.fileread
+                std.healthy
+                std.integer
+                std.ip
+                std.log
+                std.port
+                std.querysort
+                std.random
+                std.real
+                std.real2time
+                std.rollback
+                std.set_ip_tos
+                std.strstr
+                std.syslog
+                std.time
+                std.time2integer
+                std.time2real
+                std.timestamp
+                std.tolower
+                std.toupper
+                synth
+                synthetic
+            ]
+
+            BUILTIN_VARIABLES = Set.new %w[
+                bereq
+                bereq.backend
+                bereq.between_bytes_timeout
+                bereq.connect_timeout
+                bereq.first_byte_timeout
+                bereq.method
+                bereq.proto
+                bereq.retries
+                bereq.uncacheable
+                bereq.url
+                bereq.xid
+                beresp
+                beresp.age
+                beresp.backend
+                beresp.backend.ip
+                beresp.backend.name
+                beresp.do_esi
+                beresp.do_gunzip
+                beresp.do_gzip
+                beresp.do_stream
+                beresp.grace
+                beresp.keep
+                beresp.proto
+                beresp.reason
+                beresp.status
+                beresp.storage_hint
+                beresp.ttl
+                beresp.uncacheable
+                beresp.was_304
+                client.identity
+                client.ip
+                local.ip
+                now
+                obj.age
+                obj.grace
+                obj.hits
+                obj.keep
+                obj.proto
+                obj.reason
+                obj.status
+                obj.ttl
+                obj.uncacheable
+                remote.ip
+                req
+                req.backend_hint
+                req.can_gzip
+                req.esi
+                req.esi_level
+                req.hash_always_miss
+                req.hash_ignore_busy
+                req.method
+                req.proto
+                req.restarts
+                req.ttl
+                req.url
+                req.xid
+                resp
+                resp.proto
+                resp.reason
+                resp.status
+                server.hostname
+                server.identity
+                server.ip
+            ]
+
+            BUILTIN_ROUTINES = Set.new %w[
+                backend_error
+                backend_fetch
+                backend_response
+                purge
+                deliver
+                fini
+                hash
+                hit
+                init
+                miss
+                pass
+                pipe
+                recv
+                synth
+            ]
+
+            STATES_MAP = {
+                :root => Text,
+                :string => Str::Double,
+            }
+
+            state :default do
+                rule /\r\n?|\n/ do
+                    token STATES_MAP[state.name.to_sym]
+                end
+                rule /./ do
+                    token STATES_MAP[state.name.to_sym]
+                end
+            end
+
+            state :root do
+                # long strings ({" ... "})
+                rule %r'\{".*?"}'m, Str::Single
+
+                # comments
+                rule %r'/\*.*?\*/'m, Comment::Multiline
+                rule %r'(?://|#).*', Comment::Single
+
+                rule /true|false/, Keyword::Constant
+
+                # "wildcard variables"
+                rule /(?:(?:be)?re(?:sp|q)|obj)\.http\.[a-zA-Z0-9_.-]+/ do
+                    token Name::Variable
+                end
+
+                rule /(sub)(#{SPACE})([a-zA-Z0-9_-]+)/ do
+                    groups Keyword, Text, Name::Function
+                end
+
+                # inline C (C{ ... }C)
+                rule /C\{/ do
+                    token Comment::Preproc
+                    push :inline_c
+                end
+
+                rule /[a-zA-Z_.-]+/ do |m|
+                    next token Keyword if KEYWORDS.include? m[0]
+                    next token Name::Function if BUILTIN_FUNCTIONS.include? m[0]
+                    next token Name::Variable if BUILTIN_VARIABLES.include? m[0]
+                    token Text
+                end
+
+                # duration
+                rule /(?:#{LNUM}|#{DNUM})(?:ms|[smhdwy])/, Literal::Number::Other
+                # size in bytes
+                rule /#{LNUM}[KMGT]?B/, Literal::Number::Other
+                # literal numeric values (integer/float)
+                rule /#{LNUM}/, Num::Integer
+                rule /#{DNUM}/, Num::Float
+
+                # standard strings
+                rule /"/ do |m|
+                    token Str::Double
+                    push :string
+                end
+
+                rule %r'[&|+-]{2}|[<=>!*/+-]=|<<|>>|!~|[-+*/%><=!&|~]', Operator
+
+                rule /[{}();.,]/, Punctuation
+
+                mixin :default
+            end
+
+            state :string do
+                rule /"/, Str::Double, :pop!
+                rule /\\[\\"nt]/, Str::Escape
+
+                mixin :default
+            end
+
+            state :inline_c do
+                rule /}C/, Comment::Preproc, :pop!
+                rule /.*?(?=}C)/m do
+                    delegate C
+                end
+            end
+        end
+    end
+end

spec/lexers/varnish_spec.rb

@@ -0,0 +1,24 @@
+describe Rouge::Lexers::Varnish do
+  let(:subject) { Rouge::Lexers::Varnish.new }
+
+  describe 'guessing' do
+    include Support::Guessing
+
+    it 'guesses by filename' do
+      assert_guess :filename => 'varnish.vcl'
+      assert_guess :filename => 'builtin.vcl'
+    end
+
+    it 'guesses by mimetype' do
+      assert_guess :mimetype => 'text/x-varnish'
+    end
+  end
+
+  describe 'lexing' do
+    include Support::Lexing
+
+    it 'forwards C blocks to C lexer' do
+      assert_tokens_equal 'foo C{int}C bar', ['Text', 'foo '], [ 'Comment.Preproc', 'C{' ], [ 'Keyword.Type', 'int' ], [ 'Comment.Preproc', '}C' ], [ 'Text', ' bar' ] 
+    end
+  end
+end

spec/visual/samples/varnish

@@ -0,0 +1,40 @@
+# Define a function that converts a string to lower-case in-place.
+C{
+    #include <ctype.h>
+
+    static void strtolower(char *c) {
+        for (; *c; c++) {
+            if (isupper(*c)) {
+                *c = tolower(*c);
+            }
+        }
+    }
+}C
+
+sub vcl_recv {
+    if (req.http.host ~ "[A-Z]" || req.url ~ "[A-Z]") {
+        # Convert host and path to lowercase in-place (dummy C code from Varnish 3 instead of builtin std.tolower function)
+        C{
+            /* foo */
+            strtolower(VRT_GetHdr(sp, HDR_REQ, "\005host:"));
+            strtolower((char *)VRT_r_req_url(sp));
+        }C
+        # and redirect based on the fake HTTP code 701 and set new URL as reason
+        return (synth(701, "http://" + req.http.host + req.url));
+    }
+
+    # Fall-through to default
+}
+
+sub vcl_synth {
+    # Check for redirects - redirects are performed using: synth(701, "http://target-url/")
+    # Thus we piggyback the redirect target in the error response variable.
+    if (701 == resp.status) {
+        set resp.http.location = resp.reason;
+        set resp.status = 301;
+        set resp.reason = "Moved permanently";
+        return(deliver);
+    }
+
+    # Fall-through to default
+}