use the new crypto/tls QUIC Transport

.github/workflows/gotip.yml

@@ -0,0 +1,28 @@
+on: [push, pull_request]
+
+jobs:
+  integration:
+    runs-on: ${{ fromJSON(vars['GOTIP_RUNNER_UBUNTU'] || '"ubuntu-latest"') }}
+    env:
+      DEBUG: false # set this to true to export qlogs and save them as artifacts
+      TIMESCALE_FACTOR: 3
+    name: Test with gotip
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3 # need to install Go for bootstrapping
+        with:
+          go-version: '1.20.x'
+      - name: Install gotip
+        run: |
+          git clone --depth=1 https://go.googlesource.com/go $HOME/gotip
+          cd $HOME/gotip/src
+          git fetch https://go.googlesource.com/go refs/changes/15/498215/1 && git checkout FETCH_HEAD
+          ./make.bash
+          echo "GOROOT=$HOME/gotip" >> $GITHUB_ENV
+          echo "$HOME/gotip/bin:$PATH" >> $GITHUB_PATH
+      - run: go version
+      - name: set qlogger
+        if: env.DEBUG == 'true'
+        run: echo "QLOGFLAG= -qlog" >> $GITHUB_ENV
+      - name: Run self tests, using QUIC v1
+        run: go run github.com/onsi/ginkgo/v2/ginkgo -r -v -randomize-all -randomize-suites -trace integrationtests/self -- -version=1 ${{ env.QLOGFLAG }}

connection.go

@@ -52,7 +52,7 @@ type streamManager interface {
 }
 
 type cryptoStreamHandler interface {
-	RunHandshake()
+	StartHandshake() error
 	ChangeConnectionID(protocol.ConnectionID)
 	SetLargest1RTTAcked(protocol.PacketNumber) error
 	SetHandshakeConfirmed()
@@ -103,15 +103,15 @@ type connRunner interface {
 
 type handshakeRunner struct {
 	onReceivedParams    func(*wire.TransportParameters)
-	onError             func(error)
+	onReceivedReadKeys  func()
 	dropKeys            func(protocol.EncryptionLevel)
 	onHandshakeComplete func()
 }
 
 func (r *handshakeRunner) OnReceivedParams(tp *wire.TransportParameters) { r.onReceivedParams(tp) }
-func (r *handshakeRunner) OnError(e error)                               { r.onError(e) }
 func (r *handshakeRunner) DropKeys(el protocol.EncryptionLevel)          { r.dropKeys(el) }
 func (r *handshakeRunner) OnHandshakeComplete()                          { r.onHandshakeComplete() }
+func (r *handshakeRunner) OnReceivedReadKeys()                           { r.onReceivedReadKeys() }
 
 type closeError struct {
 	err       error
@@ -332,14 +332,13 @@ var newConnection = func(
 	cs := handshake.NewCryptoSetupServer(
 		initialStream,
 		handshakeStream,
+		s.oneRTTStream,
 		clientDestConnID,
-		conn.LocalAddr(),
-		conn.RemoteAddr(),
 		params,
 		&handshakeRunner{
-			onReceivedParams: s.handleTransportParameters,
-			onError:          s.closeLocal,
-			dropKeys:         s.dropEncryptionLevel,
+			onReceivedParams:   s.handleTransportParameters,
+			dropKeys:           s.dropEncryptionLevel,
+			onReceivedReadKeys: s.receivedReadKeys,
 			onHandshakeComplete: func() {
 				runner.Retire(clientDestConnID)
 				close(s.handshakeCompleteChan)
@@ -420,6 +419,7 @@ var newClientConnection = func(
 	)
 	initialStream := newCryptoStream()
 	handshakeStream := newCryptoStream()
+	oneRTTStream := newCryptoStream()
 	params := &wire.TransportParameters{
 		InitialMaxStreamDataBidiRemote: protocol.ByteCount(s.config.InitialStreamReceiveWindow),
 		InitialMaxStreamDataBidiLocal:  protocol.ByteCount(s.config.InitialStreamReceiveWindow),
@@ -450,14 +450,13 @@ var newClientConnection = func(
 	cs, clientHelloWritten := handshake.NewCryptoSetupClient(
 		initialStream,
 		handshakeStream,
+		oneRTTStream,
 		destConnID,
-		conn.LocalAddr(),
-		conn.RemoteAddr(),
 		params,
 		&handshakeRunner{
 			onReceivedParams:    s.handleTransportParameters,
-			onError:             s.closeLocal,
 			dropKeys:            s.dropEncryptionLevel,
+			onReceivedReadKeys:  s.receivedReadKeys,
 			onHandshakeComplete: func() { close(s.handshakeCompleteChan) },
 		},
 		tlsConf,
@@ -469,7 +468,7 @@ var newClientConnection = func(
 	)
 	s.clientHelloWritten = clientHelloWritten
 	s.cryptoStreamHandler = cs
-	s.cryptoStreamManager = newCryptoStreamManager(cs, initialStream, handshakeStream, newCryptoStream())
+	s.cryptoStreamManager = newCryptoStreamManager(cs, initialStream, handshakeStream, oneRTTStream)
 	s.unpacker = newPacketUnpacker(cs, s.srcConnIDLen)
 	s.packer = newPacketPacker(srcConnID, s.connIDManager.Get, initialStream, handshakeStream, s.sentPacketHandler, s.retransmissionQueue, s.RemoteAddr(), cs, s.framer, s.receivedPacketHandler, s.datagramQueue, s.perspective)
 	if len(tlsConf.ServerName) > 0 {
@@ -532,11 +531,9 @@ func (s *connection) run() error {
 
 	s.timer = *newTimer()
 
-	handshaking := make(chan struct{})
-	go func() {
-		defer close(handshaking)
-		s.cryptoStreamHandler.RunHandshake()
-	}()
+	if err := s.cryptoStreamHandler.StartHandshake(); err != nil {
+		return err
+	}
 	go func() {
 		if err := s.sendQueue.Run(); err != nil {
 			s.destroyImpl(err)
@@ -688,7 +685,6 @@ runLoop:
 	}
 
 	s.cryptoStreamHandler.Close()
-	<-handshaking
 	s.handleCloseError(&closeErr)
 	if e := (&errCloseForRecreating{}); !errors.As(closeErr.err, &e) && s.tracer != nil {
 		s.tracer.Close()
@@ -719,7 +715,9 @@ func (s *connection) supportsDatagrams() bool {
 func (s *connection) ConnectionState() ConnectionState {
 	s.connStateMutex.Lock()
 	defer s.connStateMutex.Unlock()
-	s.connState.TLS = s.cryptoStreamHandler.ConnectionState()
+	cs := s.cryptoStreamHandler.ConnectionState()
+	s.connState.TLS = cs.ConnectionState
+	s.connState.Used0RTT = cs.Used0RTT
 	return s.connState
 }
 
@@ -781,7 +779,7 @@ func (s *connection) handleHandshakeComplete() {
 	if err != nil {
 		s.closeLocal(err)
 	}
-	if ticket != nil {
+	if ticket != nil { // may be nil if session tickets are disabled via tls.Config.SessionTicketsDisabled
 		s.oneRTTStream.Write(ticket)
 		for s.oneRTTStream.HasData() {
 			s.queueControlFrame(s.oneRTTStream.PopCryptoFrame(protocol.MaxPostHandshakeCryptoFrameSize))
@@ -1379,16 +1377,13 @@ func (s *connection) handleConnectionCloseFrame(frame *wire.ConnectionCloseFrame
 }
 
 func (s *connection) handleCryptoFrame(frame *wire.CryptoFrame, encLevel protocol.EncryptionLevel) error {
-	encLevelChanged, err := s.cryptoStreamManager.HandleCryptoFrame(frame, encLevel)
-	if err != nil {
-		return err
-	}
-	if encLevelChanged {
-		// Queue all packets for decryption that have been undecryptable so far.
-		s.undecryptablePacketsToProcess = s.undecryptablePackets
-		s.undecryptablePackets = nil
-	}
-	return nil
+	return s.cryptoStreamManager.HandleCryptoFrame(frame, encLevel)
+}
+
+func (s *connection) receivedReadKeys() {
+	// Queue all packets for decryption that have been undecryptable so far.
+	s.undecryptablePacketsToProcess = s.undecryptablePackets
+	s.undecryptablePackets = nil
 }
 
 func (s *connection) handleStreamFrame(frame *wire.StreamFrame) error {
@@ -1630,11 +1625,15 @@ func (s *connection) handleCloseError(closeErr *closeError) {
 }
 
 func (s *connection) dropEncryptionLevel(encLevel protocol.EncryptionLevel) {
-	s.sentPacketHandler.DropPackets(encLevel)
-	s.receivedPacketHandler.DropPackets(encLevel)
 	if s.tracer != nil {
 		s.tracer.DroppedEncryptionLevel(encLevel)
 	}
+	s.sentPacketHandler.DropPackets(encLevel)
+	s.receivedPacketHandler.DropPackets(encLevel)
+	if err := s.cryptoStreamManager.Drop(encLevel); err != nil {
+		s.closeLocal(err)
+		return
+	}
 	if encLevel == protocol.Encryption0RTT {
 		s.streamsMap.ResetFor0RTT()
 		if err := s.connFlowController.Reset(); err != nil {
@@ -1877,6 +1876,9 @@ func (s *connection) sendPacket() (bool, error) {
 		s.framer.QueueControlFrame(&wire.DataBlockedFrame{MaximumData: offset})
 	}
 	s.windowUpdateQueue.QueueAll()
+	if cf := s.cryptoStreamManager.GetPostHandshakeData(protocol.MaxPostHandshakeCryptoFrameSize); cf != nil {
+		s.queueControlFrame(cf)
+	}
 
 	now := time.Now()
 	if !s.handshakeConfirmed {

crypto_stream.go

@@ -71,17 +71,9 @@ func (s *cryptoStreamImpl) HandleCryptoFrame(f *wire.CryptoFrame) error {
 
 // GetCryptoData retrieves data that was received in CRYPTO frames
 func (s *cryptoStreamImpl) GetCryptoData() []byte {
-	if len(s.msgBuf) < 4 {
-		return nil
-	}
-	msgLen := 4 + int(s.msgBuf[1])<<16 + int(s.msgBuf[2])<<8 + int(s.msgBuf[3])
-	if len(s.msgBuf) < msgLen {
-		return nil
-	}
-	msg := make([]byte, msgLen)
-	copy(msg, s.msgBuf[:msgLen])
-	s.msgBuf = s.msgBuf[msgLen:]
-	return msg
+	b := s.msgBuf
+	s.msgBuf = nil
+	return b
 }
 
 func (s *cryptoStreamImpl) Finish() error {

crypto_stream_manager.go

@@ -8,7 +8,7 @@ import (
 )
 
 type cryptoDataHandler interface {
-	HandleMessage([]byte, protocol.EncryptionLevel) bool
+	HandleMessage([]byte, protocol.EncryptionLevel) error
 }
 
 type cryptoStreamManager struct {
@@ -33,7 +33,7 @@ func newCryptoStreamManager(
 	}
 }
 
-func (m *cryptoStreamManager) HandleCryptoFrame(frame *wire.CryptoFrame, encLevel protocol.EncryptionLevel) (bool /* encryption level changed */, error) {
+func (m *cryptoStreamManager) HandleCryptoFrame(frame *wire.CryptoFrame, encLevel protocol.EncryptionLevel) error {
 	var str cryptoStream
 	//nolint:exhaustive // CRYPTO frames cannot be sent in 0-RTT packets.
 	switch encLevel {
@@ -44,18 +44,38 @@ func (m *cryptoStreamManager) HandleCryptoFrame(frame *wire.CryptoFrame, encLeve
 	case protocol.Encryption1RTT:
 		str = m.oneRTTStream
 	default:
-		return false, fmt.Errorf("received CRYPTO frame with unexpected encryption level: %s", encLevel)
+		return fmt.Errorf("received CRYPTO frame with unexpected encryption level: %s", encLevel)
 	}
 	if err := str.HandleCryptoFrame(frame); err != nil {
-		return false, err
+		return err
 	}
 	for {
 		data := str.GetCryptoData()
 		if data == nil {
-			return false, nil
+			return nil
 		}
-		if encLevelFinished := m.cryptoHandler.HandleMessage(data, encLevel); encLevelFinished {
-			return true, str.Finish()
+		if err := m.cryptoHandler.HandleMessage(data, encLevel); err != nil {
+			return err
 		}
 	}
 }
+
+func (m *cryptoStreamManager) GetPostHandshakeData(maxSize protocol.ByteCount) *wire.CryptoFrame {
+	if !m.oneRTTStream.HasData() {
+		return nil
+	}
+	return m.oneRTTStream.PopCryptoFrame(maxSize)
+}
+
+func (m *cryptoStreamManager) Drop(encLevel protocol.EncryptionLevel) error {
+	switch encLevel {
+	case protocol.EncryptionInitial:
+		return m.initialStream.Finish()
+	case protocol.EncryptionHandshake:
+		return m.handshakeStream.Finish()
+	case protocol.Encryption0RTT:
+		return nil
+	default:
+		panic(fmt.Sprintf("dropped unexpected encryption level: %s", encLevel))
+	}
+}