caddytls: Support external certificate Managers (like Tailscale) (#4541)

Huge thank-you to Tailscale (https://tailscale.com) for making this change possible!
This is a great feature for Caddy and Tailscale is a great fit for a standard implementation.

* caddytls: GetCertificate modules; Tailscale

* Caddyfile support for get_certificate

Also fix AP provisioning in case of empty subject list (persist loaded
module on struct, much like Issuers, to surive reprovisioning).

And implement start of HTTP cert getter, still WIP.

* Update modules/caddytls/automation.go

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

* Use tsclient package, check status for name

* Implement HTTP cert getter

And use reuse CertMagic's PEM functions for private keys.

* Remove cache option from Tailscale getter

Tailscale does its own caching and we don't need the added complexity...
for now, at least.

* Several updates

- Option to disable cert automation in auto HTTPS
- Support multiple cert managers
- Remove cache feature from cert manager modules
- Minor improvements to auto HTTPS logging

* Run go mod tidy

* Try to get certificates from Tailscale implicitly

Only for domains ending in .ts.net.

I think this is really cool!

Co-authored-by: Francis Lavoie <lavofr@gmail.com>

caddyconfig/httpcaddyfile/builtins.go

@@ -82,6 +82,7 @@ func parseBind(h Helper) ([]ConfigValue, error) {
 //         on_demand
 //         eab    <key_id> <mac_key>
 //         issuer <module_name> [...]
+//         get_certificate <module_name> [...]
 //     }
 //
 func parseTLS(h Helper) ([]ConfigValue, error) {
@@ -93,6 +94,7 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 	var keyType string
 	var internalIssuer *caddytls.InternalIssuer
 	var issuers []certmagic.Issuer
+	var certManagers []certmagic.CertificateManager
 	var onDemand bool
 
 	for h.Next() {
@@ -307,6 +309,22 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 				}
 				issuers = append(issuers, issuer)
 
+			case "get_certificate":
+				if !h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				modName := h.Val()
+				modID := "tls.get_certificate." + modName
+				unm, err := caddyfile.UnmarshalModule(h.Dispenser, modID)
+				if err != nil {
+					return nil, err
+				}
+				certManager, ok := unm.(certmagic.CertificateManager)
+				if !ok {
+					return nil, h.Errf("module %s (%T) is not a certmagic.CertificateManager", modID, unm)
+				}
+				certManagers = append(certManagers, certManager)
+
 			case "dns":
 				if !h.NextArg() {
 					return nil, h.ArgErr()
@@ -453,6 +471,12 @@ func parseTLS(h Helper) ([]ConfigValue, error) {
 			Value: true,
 		})
 	}
+	for _, certManager := range certManagers {
+		configVals = append(configVals, ConfigValue{
+			Class: "tls.cert_manager",
+			Value: certManager,
+		})
+	}
 
 	// custom certificate selection
 	if len(certSelector.AnyTag) > 0 {

caddyconfig/httpcaddyfile/httptype.go

@@ -446,13 +446,14 @@ func (st *ServerType) serversFromPairings(
 		// handle the auto_https global option
 		if autoHTTPS != "on" {
 			srv.AutoHTTPS = new(caddyhttp.AutoHTTPSConfig)
-			if autoHTTPS == "off" {
+			switch autoHTTPS {
+			case "off":
 				srv.AutoHTTPS.Disabled = true
-			}
-			if autoHTTPS == "disable_redirects" {
+			case "disable_redirects":
 				srv.AutoHTTPS.DisableRedir = true
-			}
-			if autoHTTPS == "ignore_loaded_certs" {
+			case "disable_certs":
+				srv.AutoHTTPS.DisableCerts = true
+			case "ignore_loaded_certs":
 				srv.AutoHTTPS.IgnoreLoadedCerts = true
 			}
 		}

caddyconfig/httpcaddyfile/options.go

@@ -384,8 +384,8 @@ func parseOptAutoHTTPS(d *caddyfile.Dispenser, _ interface{}) (interface{}, erro
 	if d.Next() {
 		return "", d.ArgErr()
 	}
-	if val != "off" && val != "disable_redirects" && val != "ignore_loaded_certs" {
-		return "", d.Errf("auto_https must be one of 'off', 'disable_redirects' or 'ignore_loaded_certs'")
+	if val != "off" && val != "disable_redirects" && val != "disable_certs" && val != "ignore_loaded_certs" {
+		return "", d.Errf("auto_https must be one of 'off', 'disable_redirects', 'disable_certs', or 'ignore_loaded_certs'")
 	}
 	return val, nil
 }

caddyconfig/httpcaddyfile/tlsapp.go

@@ -133,6 +133,13 @@ func (st ServerType) buildTLSApp(
 				ap.Issuers = issuers
 			}
 
+			// certificate managers
+			if certManagerVals, ok := sblock.pile["tls.cert_manager"]; ok {
+				for _, certManager := range certManagerVals {
+					certGetterName := certManager.Value.(caddy.Module).CaddyModule().ID.Name()
+					ap.ManagersRaw = append(ap.ManagersRaw, caddyconfig.JSONModuleObject(certManager.Value, "via", certGetterName, &warnings))
+				}
+			}
 			// custom bind host
 			for _, cfgVal := range sblock.pile["bind"] {
 				for _, iss := range ap.Issuers {

go.mod

@@ -3,11 +3,14 @@ module github.com/caddyserver/caddy/v2
 go 1.16
 
 require (
+	cloud.google.com/go/kms v1.1.0 // indirect
+	github.com/BurntSushi/toml v0.4.1 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.2
 	github.com/alecthomas/chroma v0.10.0
 	github.com/aryann/difflib v0.0.0-20210328193216-ff5ff6dc229b
-	github.com/caddyserver/certmagic v0.15.3
+	github.com/caddyserver/certmagic v0.15.4-0.20220217213750-797d29bcf32f
 	github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/google/cel-go v0.9.0
 	github.com/google/uuid v1.3.0
@@ -22,14 +25,17 @@ require (
 	github.com/smallstep/cli v0.18.0
 	github.com/smallstep/nosql v0.3.9
 	github.com/smallstep/truststore v0.10.1
+	github.com/tailscale/tscert v0.0.0-20220125204807-4509a5fbaf74
 	github.com/yuin/goldmark v1.4.4
 	github.com/yuin/goldmark-highlighting v0.0.0-20210516132338-9216f9c5aa01
 	go.uber.org/zap v1.20.0
 	golang.org/x/crypto v0.0.0-20210915214749-c084706c2272
 	golang.org/x/net v0.0.0-20210913180222-943fd674d43e
 	golang.org/x/term v0.0.0-20210503060354-a79de5458b56
-	google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2
+	google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2
 	google.golang.org/protobuf v1.27.1
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v2 v2.4.0
+	howett.net/plist v1.0.0 // indirect
 )