Varnish ESI with xkey

[sanhofman]

| Sulu Version |2.5.8
| PHP Version | 8.2.5

We setup Sulu with Nginx using Varnish as reverse proxy. We use both user_context and tag based invalidation (xkey). Following the documentation our VCL looks like this:

sub vcl_recv {
    call fos_tags_xkey_recv;
    call fos_user_context_recv;
    call sulu_recv;

    // # Add a Surrogate-Capability header to announce ESI support.
    set bereq.http.Surrogate-Capability = "abc=ESI/1.0";

    # Remove all cookies except the session ID (SULUSESSID)
    # Check configured session ID name in your config/packages/framework.yaml
    if (req.http.Cookie) {
        set req.http.Cookie = ";" + req.http.Cookie;
        set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
        set req.http.Cookie = regsuball(req.http.Cookie, ";(SULUSESSID)=", "; \1=");
        set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
        set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");

        if (req.http.Cookie == "") {
            # If there are no more cookies, remove the header to get page cached.
            unset req.http.Cookie;
        }
    }

    # No cache besides GET and HEAD
    if (req.method != "GET" && req.method != "HEAD") {
      return (pass);
    }

    # if Authorization or no-cache header we skip the cache
    if (req.http.Authorization || req.http.Cache-Control ~ "no-cache") {
        return (pass);
    }

    # Force the lookup, the backend must tell not to cache or vary on all
    # headers that are used to build the hash.
    return (hash);
}

...

Where ESI support is added below fos_tags_xkey_recv, fos_user_context_recv and sulu_recv.

    // # Add a Surrogate-Capability header to announce ESI support.
    set bereq.http.Surrogate-Capability = "abc=ESI/1.0";

But ESI doesn't work, after some research and support we discovered that the Surrogate-Capability header was not available in responses.

Probably because fos_tags_xkey_recv / fos_user_context_recv are bypassing our the ESI support header.

When we re-structure the VCL to:

sub vcl_backend_fetch {
    set bereq.http.Surrogate-Capability = "abc=ESI/1.0";
}

sub vcl_recv {
    call fos_tags_xkey_recv;
    call fos_user_context_recv;
    call sulu_recv;
...

The Surrogate-Capability header is properly available, resulting in ESI (including xkey and user context) working as expected.

Is there a reason the Surrogate-Capability header was added under vcl_recv instead of vcl_backend_fetch? I think it's best to also update this in documentation.

[alexander-schranz]

@sanhofman can it be that in your case the following part is missing:

sub vcl_backend_response {
    call sulu_backend_response;
}

Its included there: https://docs.sulu.io/en/2.6/cookbook/caching-with-varnish.html

and it should do then the same as your adoptions:

sulu/src/Sulu/Bundle/HttpCacheBundle/Resources/varnish/sulu.vcl

Lines 63 to 66 in 9ae2daa

	if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
	unset beresp.http.Surrogate-Control;
	set beresp.do_esi = true;
	}

[sanhofman]

@alexander-schranz no it's included, see full vcl:

# /etc/varnish/default.vcl
vcl 4.0;

include "/etc/varnish/sulu.vcl";
include "/etc/varnish/fos_user_context.vcl";
include "/etc/varnish/fos_user_context_url.vcl";
include "/etc/varnish/fos_tags_xkey.vcl";

acl invalidators {
    "localhost";
    "$VARNISH_INVALIDATOR_HOST";
}

backend default {
    .host = "$VARNISH_BACKEND_HOST";
    .port = "$VARNISH_BACKEND_PORT";
}

sub vcl_backend_fetch {
    set bereq.http.Surrogate-Capability = "abc=ESI/1.0";
}

sub vcl_recv {
    call fos_tags_xkey_recv;
    call fos_user_context_recv;
    call sulu_recv;

    # Remove all cookies except the session ID (SULUSESSID)
    # Check configured session ID name in your config/packages/framework.yaml
    if (req.http.Cookie) {
        set req.http.Cookie = ";" + req.http.Cookie;
        set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
        set req.http.Cookie = regsuball(req.http.Cookie, ";(SULUSESSID)=", "; \1=");
        set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
        set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");

        if (req.http.Cookie == "") {
            # If there are no more cookies, remove the header to get page cached.
            unset req.http.Cookie;
        }
    }

    # No cache besides GET and HEAD
    if (req.method != "GET" && req.method != "HEAD") {
      return (pass);
    }

    # if Authorization or no-cache header we skip the cache
    if (req.http.Authorization || req.http.Cache-Control ~ "no-cache") {
        return (pass);
    }

    # Force the lookup, the backend must tell not to cache or vary on all
    # headers that are used to build the hash.
    return (hash);
}

sub vcl_hash {
    call fos_user_context_hash;
}

sub vcl_backend_response {
    set beresp.grace = 2m;

    call fos_user_context_backend_response;
    call sulu_backend_response;

    // Check for ESI acknowledgement and remove Surrogate-Control header
    if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
        unset beresp.http.Surrogate-Control;
        set beresp.do_esi = true;
    }
}

sub vcl_deliver {
    call fos_tags_xkey_deliver;
    call fos_user_context_deliver;
    call sulu_deliver;

    # Include debugging info
    set resp.http.Obj-Hits = obj.hits;
    set resp.http.Seen-Cookie = req.http.Cookie;

    # set resp.http.Vary = "X-User-Context-Hash";
}