diff --git a/caddytest/integration/caddyfile_adapt/reverse_proxy_dynamic_upstreams.txt b/caddytest/integration/caddyfile_adapt/reverse_proxy_dynamic_upstreams.txt new file mode 100644 index 00000000000..2f2cbcd3eb1 --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/reverse_proxy_dynamic_upstreams.txt @@ -0,0 +1,116 @@ +:8884 { + reverse_proxy { + dynamic a foo 9000 + } + + reverse_proxy { + dynamic a { + name foo + port 9000 + refresh 5m + resolvers 8.8.8.8 8.8.4.4 + dial_timeout 2s + dial_fallback_delay 300ms + } + } +} + +:8885 { + reverse_proxy { + dynamic srv _api._tcp.example.com + } + + reverse_proxy { + dynamic srv { + service api + proto tcp + name example.com + refresh 5m + resolvers 8.8.8.8 8.8.4.4 + dial_timeout 1s + dial_fallback_delay -1s + } + } +} + +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":8884" + ], + "routes": [ + { + "handle": [ + { + "dynamic_upstreams": { + "name": "foo", + "port": "9000", + "source": "a" + }, + "handler": "reverse_proxy" + }, + { + "dynamic_upstreams": { + "dial_fallback_delay": 300000000, + "dial_timeout": 2000000000, + "name": "foo", + "port": "9000", + "refresh": 300000000000, + "resolver": { + "addresses": [ + "8.8.8.8", + "8.8.4.4" + ] + }, + "source": "a" + }, + "handler": "reverse_proxy" + } + ] + } + ] + }, + "srv1": { + "listen": [ + ":8885" + ], + "routes": [ + { + "handle": [ + { + "dynamic_upstreams": { + "name": "_api._tcp.example.com", + "source": "srv" + }, + "handler": "reverse_proxy" + }, + { + "dynamic_upstreams": { + "dial_fallback_delay": -1000000000, + "dial_timeout": 1000000000, + "name": "example.com", + "proto": "tcp", + "refresh": 300000000000, + "resolver": { + "addresses": [ + "8.8.8.8", + "8.8.4.4" + ] + }, + "service": "api", + "source": "srv" + }, + "handler": "reverse_proxy" + } + ] + } + ] + } + } + } + } +} \ No newline at end of file diff --git a/caddytest/integration/caddyfile_adapt/reverse_proxy_options.txt b/caddytest/integration/caddyfile_adapt/reverse_proxy_options.txt index 544bb9ff679..3a8ebe0e48d 100644 --- a/caddytest/integration/caddyfile_adapt/reverse_proxy_options.txt +++ b/caddytest/integration/caddyfile_adapt/reverse_proxy_options.txt @@ -17,6 +17,7 @@ https://example.com { dial_fallback_delay 5s response_header_timeout 8s expect_continue_timeout 9s + resolvers 8.8.8.8 8.8.4.4 versions h2c 2 compression off @@ -86,6 +87,12 @@ https://example.com { "max_response_header_size": 30000000, "protocol": "http", "read_buffer_size": 10000000, + "resolver": { + "addresses": [ + "8.8.8.8", + "8.8.4.4" + ] + }, "response_header_timeout": 8000000000, "versions": [ "h2c", diff --git a/modules/caddyhttp/reverseproxy/caddyfile.go b/modules/caddyhttp/reverseproxy/caddyfile.go index c7f555f8a44..7f2f9669ac9 100644 --- a/modules/caddyhttp/reverseproxy/caddyfile.go +++ b/modules/caddyhttp/reverseproxy/caddyfile.go @@ -54,6 +54,7 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) // reverse_proxy [] [] { // # upstreams // to +// dynamic [...] // // # load balancing // lb_policy [] @@ -270,6 +271,25 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { } } + case "dynamic": + if !d.NextArg() { + return d.ArgErr() + } + if h.DynamicUpstreams != nil { + return d.Err("dynamic upstreams already specified") + } + dynModule := d.Val() + modID := "http.reverse_proxy.upstreams." + dynModule + unm, err := caddyfile.UnmarshalModule(d, modID) + if err != nil { + return err + } + source, ok := unm.(UpstreamSource) + if !ok { + return d.Errf("module %s (%T) is not an UpstreamSource", modID, unm) + } + h.DynamicUpstreamsRaw = caddyconfig.JSONModuleObject(source, "source", dynModule, nil) + case "lb_policy": if !d.NextArg() { return d.ArgErr() @@ -799,6 +819,7 @@ func (h *Handler) FinalizeUnmarshalCaddyfile(helper httpcaddyfile.Helper) error // dial_fallback_delay // response_header_timeout // expect_continue_timeout +// resolvers // tls // tls_client_auth | // tls_insecure_skip_verify @@ -887,6 +908,15 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { } h.ExpectContinueTimeout = caddy.Duration(dur) + case "resolvers": + if h.Resolver == nil { + h.Resolver = new(UpstreamResolver) + } + h.Resolver.Addresses = d.RemainingArgs() + if len(h.Resolver.Addresses) == 0 { + return d.Errf("must specify at least one resolver address") + } + case "tls_client_auth": if h.TLS == nil { h.TLS = new(TLSConfig) @@ -1024,10 +1054,201 @@ func (h *HTTPTransport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { return nil } +// UnmarshalCaddyfile deserializes Caddyfile tokens into h. +// +// dynamic srv [] { +// service +// proto +// name +// refresh +// resolvers +// dial_timeout +// dial_fallback_delay +// } +// +func (u *SRVUpstreams) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + for d.Next() { + args := d.RemainingArgs() + if len(args) > 1 { + return d.ArgErr() + } + if len(args) > 0 { + u.Name = args[0] + } + + for d.NextBlock(0) { + switch d.Val() { + case "service": + if !d.NextArg() { + return d.ArgErr() + } + if u.Service != "" { + return d.Errf("srv service has already been specified") + } + u.Service = d.Val() + + case "proto": + if !d.NextArg() { + return d.ArgErr() + } + if u.Proto != "" { + return d.Errf("srv proto has already been specified") + } + u.Proto = d.Val() + + case "name": + if !d.NextArg() { + return d.ArgErr() + } + if u.Name != "" { + return d.Errf("srv name has already been specified") + } + u.Name = d.Val() + + case "refresh": + if !d.NextArg() { + return d.ArgErr() + } + dur, err := caddy.ParseDuration(d.Val()) + if err != nil { + return d.Errf("parsing refresh interval duration: %v", err) + } + u.Refresh = caddy.Duration(dur) + + case "resolvers": + if u.Resolver == nil { + u.Resolver = new(UpstreamResolver) + } + u.Resolver.Addresses = d.RemainingArgs() + if len(u.Resolver.Addresses) == 0 { + return d.Errf("must specify at least one resolver address") + } + + case "dial_timeout": + if !d.NextArg() { + return d.ArgErr() + } + dur, err := caddy.ParseDuration(d.Val()) + if err != nil { + return d.Errf("bad timeout value '%s': %v", d.Val(), err) + } + u.DialTimeout = caddy.Duration(dur) + + case "dial_fallback_delay": + if !d.NextArg() { + return d.ArgErr() + } + dur, err := caddy.ParseDuration(d.Val()) + if err != nil { + return d.Errf("bad delay value '%s': %v", d.Val(), err) + } + u.FallbackDelay = caddy.Duration(dur) + + default: + return d.Errf("unrecognized srv option '%s'", d.Val()) + } + } + } + + return nil +} + +// UnmarshalCaddyfile deserializes Caddyfile tokens into h. +// +// dynamic a [ +// port +// refresh +// resolvers +// dial_timeout +// dial_fallback_delay +// } +// +func (u *AUpstreams) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + for d.Next() { + args := d.RemainingArgs() + if len(args) > 2 { + return d.ArgErr() + } + if len(args) > 0 { + u.Name = args[0] + u.Port = args[1] + } + + for d.NextBlock(0) { + switch d.Val() { + case "name": + if !d.NextArg() { + return d.ArgErr() + } + if u.Name != "" { + return d.Errf("a name has already been specified") + } + u.Name = d.Val() + + case "port": + if !d.NextArg() { + return d.ArgErr() + } + if u.Port != "" { + return d.Errf("a port has already been specified") + } + u.Port = d.Val() + + case "refresh": + if !d.NextArg() { + return d.ArgErr() + } + dur, err := caddy.ParseDuration(d.Val()) + if err != nil { + return d.Errf("parsing refresh interval duration: %v", err) + } + u.Refresh = caddy.Duration(dur) + + case "resolvers": + if u.Resolver == nil { + u.Resolver = new(UpstreamResolver) + } + u.Resolver.Addresses = d.RemainingArgs() + if len(u.Resolver.Addresses) == 0 { + return d.Errf("must specify at least one resolver address") + } + + case "dial_timeout": + if !d.NextArg() { + return d.ArgErr() + } + dur, err := caddy.ParseDuration(d.Val()) + if err != nil { + return d.Errf("bad timeout value '%s': %v", d.Val(), err) + } + u.DialTimeout = caddy.Duration(dur) + + case "dial_fallback_delay": + if !d.NextArg() { + return d.ArgErr() + } + dur, err := caddy.ParseDuration(d.Val()) + if err != nil { + return d.Errf("bad delay value '%s': %v", d.Val(), err) + } + u.FallbackDelay = caddy.Duration(dur) + + default: + return d.Errf("unrecognized srv option '%s'", d.Val()) + } + } + } + + return nil +} + const matcherPrefix = "@" // Interface guards var ( _ caddyfile.Unmarshaler = (*Handler)(nil) _ caddyfile.Unmarshaler = (*HTTPTransport)(nil) + _ caddyfile.Unmarshaler = (*SRVUpstreams)(nil) + _ caddyfile.Unmarshaler = (*AUpstreams)(nil) ) diff --git a/modules/caddyhttp/reverseproxy/httptransport.go b/modules/caddyhttp/reverseproxy/httptransport.go index 35bc94783e6..42a9a22d56e 100644 --- a/modules/caddyhttp/reverseproxy/httptransport.go +++ b/modules/caddyhttp/reverseproxy/httptransport.go @@ -153,15 +153,9 @@ func (h *HTTPTransport) NewTransport(ctx caddy.Context) (*http.Transport, error) } if h.Resolver != nil { - for _, v := range h.Resolver.Addresses { - addr, err := caddy.ParseNetworkAddress(v) - if err != nil { - return nil, err - } - if addr.PortRangeSize() != 1 { - return nil, fmt.Errorf("resolver address must have exactly one address; cannot call %v", addr) - } - h.Resolver.netAddrs = append(h.Resolver.netAddrs, addr) + err := h.Resolver.ParseAddresses() + if err != nil { + return nil, err } d := &net.Dialer{ Timeout: time.Duration(h.DialTimeout), @@ -388,18 +382,6 @@ func (t TLSConfig) MakeTLSClientConfig(ctx caddy.Context) (*tls.Config, error) { return cfg, nil } -// UpstreamResolver holds the set of addresses of DNS resolvers of -// upstream addresses -type UpstreamResolver struct { - // The addresses of DNS resolvers to use when looking up the addresses of proxy upstreams. - // It accepts [network addresses](/docs/conventions#network-addresses) - // with port range of only 1. If the host is an IP address, it will be dialed directly to resolve the upstream server. - // If the host is not an IP address, the addresses are resolved using the [name resolution convention](https://golang.org/pkg/net/#hdr-Name_Resolution) of the Go standard library. - // If the array contains more than 1 resolver address, one is chosen at random. - Addresses []string `json:"addresses,omitempty"` - netAddrs []caddy.NetworkAddress -} - // KeepAlive holds configuration pertaining to HTTP Keep-Alive. type KeepAlive struct { // Whether HTTP Keep-Alive is enabled. Default: true diff --git a/modules/caddyhttp/reverseproxy/upstreams.go b/modules/caddyhttp/reverseproxy/upstreams.go index b1983775bf9..91c567dcb1e 100644 --- a/modules/caddyhttp/reverseproxy/upstreams.go +++ b/modules/caddyhttp/reverseproxy/upstreams.go @@ -1,7 +1,9 @@ package reverseproxy import ( + "context" "fmt" + weakrand "math/rand" "net" "net/http" "strconv" @@ -30,10 +32,6 @@ func init() { // // Returned upstreams are sorted by priority and weight. type SRVUpstreams struct { - // The interval at which to refresh the SRV lookup. - // Results are cached between lookups. Default: 1m - Refresh time.Duration `json:"refresh,omitempty"` - // The service label. Service string `json:"service,omitempty"` @@ -44,6 +42,25 @@ type SRVUpstreams struct { // empty, the entire domain name to look up. Name string `json:"name,omitempty"` + // The interval at which to refresh the SRV lookup. + // Results are cached between lookups. Default: 1m + Refresh caddy.Duration `json:"refresh,omitempty"` + + // Configures the DNS resolver used to resolve the + // SRV address to SRV records. + Resolver *UpstreamResolver `json:"resolver,omitempty"` + + // If Resolver is configured, how long to wait before + // timing out trying to connect to the DNS server. + DialTimeout caddy.Duration `json:"dial_timeout,omitempty"` + + // If Resolver is configured, how long to wait before + // spawning an RFC 6555 Fast Fallback connection. + // A negative value disables this. + FallbackDelay caddy.Duration `json:"dial_fallback_delay,omitempty"` + + resolver *net.Resolver + logger *zap.Logger } @@ -62,12 +79,32 @@ func (su SRVUpstreams) String() string { func (su *SRVUpstreams) Provision(ctx caddy.Context) error { su.logger = ctx.Logger(su) - if su.Proto != "tcp" && su.Proto != "udp" { - return fmt.Errorf("invalid proto '%s'", su.Proto) - } if su.Refresh == 0 { - su.Refresh = time.Minute + su.Refresh = caddy.Duration(time.Minute) + } + + if su.Resolver != nil { + err := su.Resolver.ParseAddresses() + if err != nil { + return err + } + d := &net.Dialer{ + Timeout: time.Duration(su.DialTimeout), + FallbackDelay: time.Duration(su.FallbackDelay), + } + su.resolver = &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, _, _ string) (net.Conn, error) { + //nolint:gosec + addr := su.Resolver.netAddrs[weakrand.Intn(len(su.Resolver.netAddrs))] + return d.DialContext(ctx, addr.Network, addr.JoinHostPort(0)) + }, + } + } + if su.resolver == nil { + su.resolver = net.DefaultResolver } + return nil } @@ -105,7 +142,8 @@ func (su SRVUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) { zap.String("proto", proto), zap.String("name", name)) - _, records, err := net.DefaultResolver.LookupSRV(r.Context(), service, proto, name) + _, records, err := su.resolver.LookupSRV(r.Context(), service, proto, name) + if err != nil { // From LookupSRV docs: "If the response contains invalid names, those records are filtered // out and an error will be returned alongside the the remaining results, if any." Thus, we @@ -153,7 +191,7 @@ type srvLookup struct { } func (sl srvLookup) isFresh() bool { - return time.Since(sl.freshness) < sl.srvUpstreams.Refresh + return time.Since(sl.freshness) < time.Duration(sl.srvUpstreams.Refresh) } var ( @@ -171,9 +209,24 @@ type AUpstreams struct { // The port to use with the upstreams. Default: 80 Port string `json:"port,omitempty"` - // The interval at which to refresh the SRV lookup. + // The interval at which to refresh the A lookup. // Results are cached between lookups. Default: 1m - Refresh time.Duration `json:"refresh,omitempty"` + Refresh caddy.Duration `json:"refresh,omitempty"` + + // Configures the DNS resolver used to resolve the + // domain name to A records. + Resolver *UpstreamResolver `json:"resolver,omitempty"` + + // If Resolver is configured, how long to wait before + // timing out trying to connect to the DNS server. + DialTimeout caddy.Duration `json:"dial_timeout,omitempty"` + + // If Resolver is configured, how long to wait before + // spawning an RFC 6555 Fast Fallback connection. + // A negative value disables this. + FallbackDelay caddy.Duration `json:"dial_fallback_delay,omitempty"` + + resolver *net.Resolver } // CaddyModule returns the Caddy module information. @@ -188,11 +241,34 @@ func (au AUpstreams) String() string { return au.Name } func (au *AUpstreams) Provision(_ caddy.Context) error { if au.Refresh == 0 { - au.Refresh = time.Minute + au.Refresh = caddy.Duration(time.Minute) } if au.Port == "" { au.Port = "80" } + + if au.Resolver != nil { + err := au.Resolver.ParseAddresses() + if err != nil { + return err + } + d := &net.Dialer{ + Timeout: time.Duration(au.DialTimeout), + FallbackDelay: time.Duration(au.FallbackDelay), + } + au.resolver = &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, _, _ string) (net.Conn, error) { + //nolint:gosec + addr := au.Resolver.netAddrs[weakrand.Intn(len(au.Resolver.netAddrs))] + return d.DialContext(ctx, addr.Network, addr.JoinHostPort(0)) + }, + } + } + if au.resolver == nil { + au.resolver = net.DefaultResolver + } + return nil } @@ -223,7 +299,7 @@ func (au AUpstreams) GetUpstreams(r *http.Request) ([]*Upstream, error) { name := repl.ReplaceAll(au.Name, "") port := repl.ReplaceAll(au.Port, "") - ips, err := net.DefaultResolver.LookupIPAddr(r.Context(), name) + ips, err := au.resolver.LookupIPAddr(r.Context(), name) if err != nil { return nil, err } @@ -259,7 +335,35 @@ type aLookup struct { } func (al aLookup) isFresh() bool { - return time.Since(al.freshness) < al.aUpstreams.Refresh + return time.Since(al.freshness) < time.Duration(al.aUpstreams.Refresh) +} + +// UpstreamResolver holds the set of addresses of DNS resolvers of +// upstream addresses +type UpstreamResolver struct { + // The addresses of DNS resolvers to use when looking up the addresses of proxy upstreams. + // It accepts [network addresses](/docs/conventions#network-addresses) + // with port range of only 1. If the host is an IP address, it will be dialed directly to resolve the upstream server. + // If the host is not an IP address, the addresses are resolved using the [name resolution convention](https://golang.org/pkg/net/#hdr-Name_Resolution) of the Go standard library. + // If the array contains more than 1 resolver address, one is chosen at random. + Addresses []string `json:"addresses,omitempty"` + netAddrs []caddy.NetworkAddress +} + +// ParseAddresses parses all the configured network addresses +// and ensures they're ready to be used. +func (u *UpstreamResolver) ParseAddresses() error { + for _, v := range u.Addresses { + addr, err := caddy.ParseNetworkAddress(v) + if err != nil { + return err + } + if addr.PortRangeSize() != 1 { + return fmt.Errorf("resolver address must have exactly one address; cannot call %v", addr) + } + u.netAddrs = append(u.netAddrs, addr) + } + return nil } var (