diff --git a/http3/server.go b/http3/server.go
index c568decbdf3..9b5b6a64203 100644
--- a/http3/server.go
+++ b/http3/server.go
@@ -12,7 +12,6 @@ import (
 	"runtime"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/lucas-clemente/quic-go"
@@ -117,6 +116,11 @@ func newConnError(code errorCode, err error) requestError {
 	return requestError{err: err, connErr: code}
 }
 
+// listenerInfo contains info about specific listener added with addListener
+type listenerInfo struct {
+	port int // 0 means that no info about port is available
+}
+
 // Server is a HTTP/3 server.
 type Server struct {
 	*http.Server
@@ -134,12 +138,14 @@ type Server struct {
 	// If needed Port can be manually set when the Server is created.
 	// This is useful when a Layer 4 firewall is redirecting UDP traffic and clients must use
 	// a port different from the port the Server is listening on.
-	Port uint32
+	Port int
 
-	mutex     sync.Mutex
-	listeners map[*quic.EarlyListener]struct{}
+	mutex     sync.RWMutex
+	listeners map[*quic.EarlyListener]listenerInfo
 	closed    utils.AtomicBool
 
+	altSvcHeader string
+
 	loggerOnce sync.Once
 	logger     utils.Logger
 }
@@ -237,21 +243,94 @@ func (s *Server) serveImpl(startListener func() (quic.EarlyListener, error)) err
 	}
 }
 
+func extractPort(addr string) (int, error) {
+	_, portStr, err := net.SplitHostPort(addr)
+	if err != nil {
+		return 0, err
+	}
+
+	portInt, err := net.LookupPort("tcp", portStr)
+	if err != nil {
+		return 0, err
+	}
+	return portInt, nil
+}
+
+func (s *Server) generateAltSvcHeader() {
+	if len(s.listeners) == 0 {
+		// Don't announce any ports since no one is listening for connections
+		s.altSvcHeader = ""
+		return
+	}
+
+	// This code assumes that we will use protocol.SupportedVersions if no quic.Config is passed.
+	supportedVersions := protocol.SupportedVersions
+	if s.QuicConfig != nil && len(s.QuicConfig.Versions) > 0 {
+		supportedVersions = s.QuicConfig.Versions
+	}
+	var versionStrings []string
+	for _, version := range supportedVersions {
+		if v := versionToALPN(version); len(v) > 0 {
+			versionStrings = append(versionStrings, v)
+		}
+	}
+
+	var altSvc []string
+	addPort := func(port int) {
+		for _, v := range versionStrings {
+			altSvc = append(altSvc, fmt.Sprintf(`%s=":%d"; ma=2592000`, v, port))
+		}
+	}
+
+	if s.Port != 0 {
+		// if Port is specified, we must use it instead of the
+		// listener addresses since there's a reason it's specified.
+		addPort(s.Port)
+	} else {
+		// if we have some listeners assigned, try to find ports
+		// which we can announce, otherwise nothing should be announced
+		validPortsFound := false
+		for _, info := range s.listeners {
+			if info.port != 0 {
+				addPort(info.port)
+				validPortsFound = true
+			}
+		}
+		if !validPortsFound {
+			if port, err := extractPort(s.Addr); err == nil {
+				addPort(port)
+			}
+		}
+	}
+
+	s.altSvcHeader = strings.Join(altSvc, ",")
+}
+
 // We store a pointer to interface in the map set. This is safe because we only
 // call trackListener via Serve and can track+defer untrack the same pointer to
 // local variable there. We never need to compare a Listener from another caller.
 func (s *Server) addListener(l *quic.EarlyListener) {
 	s.mutex.Lock()
 	if s.listeners == nil {
-		s.listeners = make(map[*quic.EarlyListener]struct{})
+		s.listeners = make(map[*quic.EarlyListener]listenerInfo)
 	}
-	s.listeners[l] = struct{}{}
+
+	if port, err := extractPort((*l).Addr().String()); err == nil {
+		s.listeners[l] = listenerInfo{port}
+	} else {
+		s.logger.Errorf(
+			"Unable to extract port from listener %+v, will not be announced using SetQuicHeaders: %s", err)
+		s.listeners[l] = listenerInfo{}
+	}
+	s.generateAltSvcHeader()
+
 	s.mutex.Unlock()
 }
 
 func (s *Server) removeListener(l *quic.EarlyListener) {
 	s.mutex.Lock()
 	delete(s.listeners, l)
+	s.generateAltSvcHeader()
 	s.mutex.Unlock()
 }
 
@@ -462,39 +541,28 @@ func (s *Server) CloseGracefully(timeout time.Duration) error {
 	return nil
 }
 
-// SetQuicHeaders can be used to set the proper headers that announce that this server supports QUIC.
-// The values that are set depend on the port information from s.Server.Addr, and currently look like this (if Addr has port 443):
-//  Alt-Svc: quic=":443"; ma=2592000; v="33,32,31,30"
+// ErrNoAltSvcPort is the error returned by SetQuicHeaders when no port was found
+// for Alt-Svc to announce. This can happen if listening on a PacketConn without a port
+// (UNIX socket, for example) and no port is specified in Server.Port or Server.Addr.
+var ErrNoAltSvcPort = errors.New("no port can be announced, specify it explicitly using Server.Port or Server.Addr")
+
+// SetQuicHeaders can be used to set the proper headers that announce that this server supports HTTP/3.
+// The values set by default advertise all of the ports the server is listening on, but can be
+// changed to a specific port by setting Server.Port before launching the serverr.
+// If no listener's Addr().String() returns an address with a valid port, Server.Addr will be used
+// to extract the port, if specified.
+// For example, a server launched using ListenAndServe on an address with port 443 would set:
+// 	Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000
 func (s *Server) SetQuicHeaders(hdr http.Header) error {
-	port := atomic.LoadUint32(&s.Port)
+	s.mutex.RLock()
+	defer s.mutex.RUnlock()
 
-	if port == 0 {
-		// Extract port from s.Server.Addr
-		_, portStr, err := net.SplitHostPort(s.Server.Addr)
-		if err != nil {
-			return err
-		}
-		portInt, err := net.LookupPort("tcp", portStr)
-		if err != nil {
-			return err
-		}
-		port = uint32(portInt)
-		atomic.StoreUint32(&s.Port, port)
-	}
-
-	// This code assumes that we will use protocol.SupportedVersions if no quic.Config is passed.
-	supportedVersions := protocol.SupportedVersions
-	if s.QuicConfig != nil && len(s.QuicConfig.Versions) > 0 {
-		supportedVersions = s.QuicConfig.Versions
-	}
-	altSvc := make([]string, 0, len(supportedVersions))
-	for _, version := range supportedVersions {
-		v := versionToALPN(version)
-		if len(v) > 0 {
-			altSvc = append(altSvc, fmt.Sprintf(`%s=":%d"; ma=2592000`, v, port))
-		}
+	if s.altSvcHeader == "" {
+		return ErrNoAltSvcPort
 	}
-	hdr.Add("Alt-Svc", strings.Join(altSvc, ","))
+	// use the map directly to avoid constant canonicalization
+	// since the key is already canonicalized
+	hdr["Alt-Svc"] = append(hdr["Alt-Svc"], s.altSvcHeader)
 	return nil
 }
 
diff --git a/http3/server_test.go b/http3/server_test.go
index 0ff6bdf8e95..ffacba0ab71 100644
--- a/http3/server_test.go
+++ b/http3/server_test.go
@@ -24,6 +24,7 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	gmtypes "github.com/onsi/gomega/types"
 )
 
 type mockConn struct {
@@ -39,6 +40,49 @@ func (c *mockConn) GetQUICVersion() protocol.VersionNumber {
 	return c.version
 }
 
+type mockAddr struct {
+	addr string
+}
+
+func (ma *mockAddr) Network() string {
+	return "udp"
+}
+
+func (ma *mockAddr) String() string {
+	return ma.addr
+}
+
+type mockAddrListener struct {
+	*mockquic.MockEarlyListener
+	addr *mockAddr
+}
+
+func (m *mockAddrListener) Addr() net.Addr {
+	_ = m.MockEarlyListener.Addr()
+	return m.addr
+}
+
+func newMockAddrListener(addr string) *mockAddrListener {
+	return &mockAddrListener{
+		MockEarlyListener: mockquic.NewMockEarlyListener(mockCtrl),
+		addr: &mockAddr{
+			addr: addr,
+		},
+	}
+}
+
+type noPortListener struct {
+	*mockAddrListener
+}
+
+func (m *noPortListener) Addr() net.Addr {
+	_ = m.mockAddrListener.Addr()
+	return &net.UnixAddr{
+		Net:  "unix",
+		Name: "/tmp/quic.sock",
+	}
+}
+
 var _ = Describe("Server", func() {
 	var (
 		s                  *Server
@@ -550,55 +594,100 @@ var _ = Describe("Server", func() {
 			s.QuicConfig = &quic.Config{Versions: []protocol.VersionNumber{protocol.VersionDraft29}}
 		})
 
+		var ln1 quic.EarlyListener
+		var ln2 quic.EarlyListener
 		expected := http.Header{
 			"Alt-Svc": {`h3-29=":443"; ma=2592000`},
 		}
 
-		It("sets proper headers with numeric port", func() {
-			s.Server.Addr = ":443"
+		addListener := func(addr string, ln *quic.EarlyListener) {
+			mln := newMockAddrListener(addr)
+			mln.EXPECT().Addr()
+			*ln = mln
+			s.addListener(ln)
+		}
+
+		removeListener := func(ln *quic.EarlyListener) {
+			s.removeListener(ln)
+		}
+
+		checkSetHeaders := func(expected gmtypes.GomegaMatcher) {
 			hdr := http.Header{}
 			Expect(s.SetQuicHeaders(hdr)).To(Succeed())
-			Expect(hdr).To(Equal(expected))
+			Expect(hdr).To(expected)
+		}
+
+		checkSetHeaderError := func() {
+			hdr := http.Header{}
+			Expect(s.SetQuicHeaders(hdr)).To(Equal(ErrNoAltSvcPort))
+		}
+
+		It("sets proper headers with numeric port", func() {
+			addListener(":443", &ln1)
+			checkSetHeaders(Equal(expected))
+			removeListener(&ln1)
+			checkSetHeaderError()
 		})
 
 		It("sets proper headers with full addr", func() {
-			s.Server.Addr = "127.0.0.1:443"
-			hdr := http.Header{}
-			Expect(s.SetQuicHeaders(hdr)).To(Succeed())
-			Expect(hdr).To(Equal(expected))
+			addListener("127.0.0.1:443", &ln1)
+			checkSetHeaders(Equal(expected))
+			removeListener(&ln1)
+			checkSetHeaderError()
 		})
 
 		It("sets proper headers with string port", func() {
-			s.Server.Addr = ":https"
-			hdr := http.Header{}
-			Expect(s.SetQuicHeaders(hdr)).To(Succeed())
-			Expect(hdr).To(Equal(expected))
+			addListener(":https", &ln1)
+			checkSetHeaders(Equal(expected))
+			removeListener(&ln1)
+			checkSetHeaderError()
 		})
 
 		It("works multiple times", func() {
-			s.Server.Addr = ":https"
-			hdr := http.Header{}
-			Expect(s.SetQuicHeaders(hdr)).To(Succeed())
-			Expect(hdr).To(Equal(expected))
-			hdr = http.Header{}
-			Expect(s.SetQuicHeaders(hdr)).To(Succeed())
-			Expect(hdr).To(Equal(expected))
+			addListener(":https", &ln1)
+			checkSetHeaders(Equal(expected))
+			checkSetHeaders(Equal(expected))
+			removeListener(&ln1)
+			checkSetHeaderError()
 		})
 
 		It("works if the quic.Config sets QUIC versions", func() {
-			s.Server.Addr = ":443"
 			s.QuicConfig.Versions = []quic.VersionNumber{quic.Version1, quic.VersionDraft29}
-			hdr := http.Header{}
-			Expect(s.SetQuicHeaders(hdr)).To(Succeed())
-			Expect(hdr).To(Equal(http.Header{"Alt-Svc": {`h3=":443"; ma=2592000,h3-29=":443"; ma=2592000`}}))
+			addListener(":443", &ln1)
+			checkSetHeaders(Equal(http.Header{"Alt-Svc": {`h3=":443"; ma=2592000,h3-29=":443"; ma=2592000`}}))
+			removeListener(&ln1)
+			checkSetHeaderError()
 		})
 
 		It("uses s.Port if set to a non-zero value", func() {
-			s.Server.Addr = ":443"
 			s.Port = 8443
-			hdr := http.Header{}
-			Expect(s.SetQuicHeaders(hdr)).To(Succeed())
-			Expect(hdr).To(Equal(http.Header{"Alt-Svc": {`h3-29=":8443"; ma=2592000`}}))
+			addListener(":443", &ln1)
+			checkSetHeaders(Equal(http.Header{"Alt-Svc": {`h3-29=":8443"; ma=2592000`}}))
+			removeListener(&ln1)
+			checkSetHeaderError()
+		})
+
+		It("uses s.Addr if listeners don't have ports available", func() {
+			s.Addr = ":443"
+			mln := &noPortListener{newMockAddrListener("")}
+			mln.EXPECT().Addr()
+			ln1 = mln
+			s.addListener(&ln1)
+			checkSetHeaders(Equal(expected))
+			s.removeListener(&ln1)
+			checkSetHeaderError()
+		})
+
+		It("properly announces multiple listeners", func() {
+			addListener(":443", &ln1)
+			addListener(":8443", &ln2)
+			checkSetHeaders(Or(
+				Equal(http.Header{"Alt-Svc": {`h3-29=":443"; ma=2592000,h3-29=":8443"; ma=2592000`}}),
+				Equal(http.Header{"Alt-Svc": {`h3-29=":8443"; ma=2592000,h3-29=":443"; ma=2592000`}}),
+			))
+			removeListener(&ln1)
+			removeListener(&ln2)
+			checkSetHeaderError()
 		})
 	})
 
@@ -657,7 +746,7 @@ var _ = Describe("Server", func() {
 		})
 
 		It("serves a packet conn", func() {
-			ln := mockquic.NewMockEarlyListener(mockCtrl)
+			ln := newMockAddrListener(":443")
 			conn := &net.UDPConn{}
 			quicListen = func(c net.PacketConn, tlsConf *tls.Config, config *quic.Config) (quic.EarlyListener, error) {
 				Expect(c).To(Equal(conn))
@@ -672,6 +761,7 @@ var _ = Describe("Server", func() {
 				<-stopAccept
 				return nil, errors.New("closed")
 			})
+			ln.EXPECT().Addr() // generate alt-svc headers
 			done := make(chan struct{})
 			go func() {
 				defer GinkgoRecover()
@@ -686,8 +776,8 @@ var _ = Describe("Server", func() {
 		})
 
 		It("serves two packet conns", func() {
-			ln1 := mockquic.NewMockEarlyListener(mockCtrl)
-			ln2 := mockquic.NewMockEarlyListener(mockCtrl)
+			ln1 := newMockAddrListener(":443")
+			ln2 := newMockAddrListener(":8443")
 			lns := make(chan quic.EarlyListener, 2)
 			lns <- ln1
 			lns <- ln2
@@ -705,11 +795,13 @@ var _ = Describe("Server", func() {
 				<-stopAccept1
 				return nil, errors.New("closed")
 			})
+			ln1.EXPECT().Addr() // generate alt-svc headers
 			stopAccept2 := make(chan struct{})
 			ln2.EXPECT().Accept(gomock.Any()).DoAndReturn(func(context.Context) (quic.Session, error) {
 				<-stopAccept2
 				return nil, errors.New("closed")
 			})
+			ln2.EXPECT().Addr()
 
 			done1 := make(chan struct{})
 			go func() {
@@ -743,7 +835,7 @@ var _ = Describe("Server", func() {
 
 		It("serves a listener", func() {
 			var called int32
-			ln := mockquic.NewMockEarlyListener(mockCtrl)
+			ln := newMockAddrListener(":443")
 			quicListen = func(conn net.PacketConn, tlsConf *tls.Config, config *quic.Config) (quic.EarlyListener, error) {
 				atomic.StoreInt32(&called, 1)
 				return ln, nil
@@ -757,6 +849,7 @@ var _ = Describe("Server", func() {
 				<-stopAccept
 				return nil, errors.New("closed")
 			})
+			ln.EXPECT().Addr() // generate alt-svc headers
 			done := make(chan struct{})
 			go func() {
 				defer GinkgoRecover()
@@ -773,8 +866,8 @@ var _ = Describe("Server", func() {
 
 		It("serves two listeners", func() {
 			var called int32
-			ln1 := mockquic.NewMockEarlyListener(mockCtrl)
-			ln2 := mockquic.NewMockEarlyListener(mockCtrl)
+			ln1 := newMockAddrListener(":443")
+			ln2 := newMockAddrListener(":8443")
 			lns := make(chan quic.EarlyListener, 2)
 			lns <- ln1
 			lns <- ln2
@@ -791,11 +884,13 @@ var _ = Describe("Server", func() {
 				<-stopAccept1
 				return nil, errors.New("closed")
 			})
+			ln1.EXPECT().Addr() // generate alt-svc headers
 			stopAccept2 := make(chan struct{})
 			ln2.EXPECT().Accept(gomock.Any()).DoAndReturn(func(context.Context) (quic.Session, error) {
 				<-stopAccept2
 				return nil, errors.New("closed")
 			})
+			ln2.EXPECT().Addr()
 
 			done1 := make(chan struct{})
 			go func() {