adding h2c graceful shutdown

modules/caddyhttp/app.go

@@ -373,9 +373,13 @@ func (app *App) Start() error {
 		// enable H2C if configured
 		if srv.protocol("h2c") {
 			h2server := &http2.Server{
-				IdleTimeout: time.Duration(srv.IdleTimeout),
+				NewWriteScheduler: func() http2.WriteScheduler { return http2.NewPriorityWriteScheduler(nil) },
 			}
-			srv.server.Handler = h2c.NewHandler(srv, h2server)
+			//nolint:errcheck
+			http2.ConfigureServer(srv.server, h2server)
+			h2chandler := newH2cHandler(h2c.NewHandler(srv, h2server))
+			srv.server.Handler = h2chandler
+			srv.h2chandler = h2chandler
 		}
 
 		for _, lnAddr := range srv.Listen {
@@ -570,12 +574,25 @@ func (app *App) Stop() error {
 				zap.Strings("addresses", server.Listen))
 		}
 	}
+	stopH2chandler := func(server *Server) {
+		defer finishedShutdown.Done()
+		startedShutdown.Done()
+
+		if server.h2chandler != nil {
+			if err := server.h2chandler.Shutdown(ctx); err != nil {
+				app.logger.Error("h2c handler shutdown",
+					zap.Error(err),
+					zap.Strings("addresses", server.Listen))
+			}
+		}
+	}
 
 	for _, server := range app.Servers {
-		startedShutdown.Add(2)
-		finishedShutdown.Add(2)
+		startedShutdown.Add(3)
+		finishedShutdown.Add(3)
 		go stopServer(server)
 		go stopH3Server(server)
+		go stopH2chandler(server)
 	}
 
 	// block until all the goroutines have been run by the scheduler;

modules/caddyhttp/h2chandler.go

@@ -0,0 +1,80 @@
+package caddyhttp
+
+import (
+	"context"
+	"math/rand"
+	"net/http"
+	"net/textproto"
+	"sync/atomic"
+	"time"
+
+	"golang.org/x/net/http/httpguts"
+)
+
+// h2chandler is a Handler which counts possible h2c upgrade requests
+type h2chandler struct {
+	cnt     uint64
+	Handler http.Handler
+}
+
+// NewH2cHandler returns an http.Handler that tracks possible h2c upgrade requests.
+func newH2cHandler(h http.Handler) *h2chandler {
+	return &h2chandler{
+		Handler: h,
+	}
+}
+
+const shutdownPollIntervalMax = 500 * time.Millisecond
+
+// Shutdown mirrors stdlib http.Server Shutdown behavior, because h2 connections are always marked active, there is no closing to be done.
+func (h *h2chandler) Shutdown(ctx context.Context) error {
+	pollIntervalBase := time.Millisecond
+	nextPollInterval := func() time.Duration {
+		// Add 10% jitter.
+		interval := pollIntervalBase + time.Duration(rand.Intn(int(pollIntervalBase/10)))
+		// Double and clamp for next time.
+		pollIntervalBase *= 2
+		if pollIntervalBase > shutdownPollIntervalMax {
+			pollIntervalBase = shutdownPollIntervalMax
+		}
+		return interval
+	}
+
+	timer := time.NewTimer(nextPollInterval())
+	defer timer.Stop()
+	for {
+		if atomic.LoadUint64(&h.cnt) == 0 {
+			return nil
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-timer.C:
+			timer.Reset(nextPollInterval())
+		}
+	}
+}
+
+// isH2cUpgrade check whether request is h2c upgrade request, copied from golang.org/x/net/http2/h2c
+func isH2cUpgrade(r *http.Request) bool {
+	if r.Method == "PRI" && len(r.Header) == 0 && r.URL.Path == "*" && r.Proto == "HTTP/2.0" {
+		return true
+	}
+
+	if httpguts.HeaderValuesContainsToken(r.Header[textproto.CanonicalMIMEHeaderKey("Upgrade")], "h2c") &&
+		httpguts.HeaderValuesContainsToken(r.Header[textproto.CanonicalMIMEHeaderKey("Connection")], "HTTP2-Settings") {
+		return true
+	}
+
+	return false
+}
+
+// ServeHTTP records underlying connections that are likely to be h2c.
+func (h *h2chandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if isH2cUpgrade(r) {
+		atomic.AddUint64(&h.cnt, 1)
+		defer atomic.AddUint64(&h.cnt, ^uint64(0))
+	}
+	h.Handler.ServeHTTP(w, r)
+	return
+}

modules/caddyhttp/server.go

@@ -175,6 +175,8 @@ type Server struct {
 	h3listeners []net.PacketConn // TODO: we have to hold these because quic-go won't close listeners it didn't create
 	addresses   []caddy.NetworkAddress
 
+	h2chandler *h2chandler
+
 	shutdownAt   time.Time
 	shutdownAtMu *sync.RWMutex