New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Fix race between activeStreams and bdp window size #5494
Changes from 5 commits
4459a50
f7515ef
ad70290
2465c21
ff5bade
30cc71c
06f7a92
067003e
9af3575
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -685,7 +685,6 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, | |
cleanup(err) | ||
return err | ||
} | ||
t.activeStreams[id] = s | ||
if channelz.IsOn() { | ||
atomic.AddInt64(&t.czData.streamsStarted, 1) | ||
atomic.StoreInt64(&t.czData.lastStreamCreatedTime, time.Now().UnixNano()) | ||
|
@@ -719,6 +718,11 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, | |
t.nextID += 2 | ||
s.id = h.streamID | ||
s.fc = &inFlow{limit: uint32(t.initialWindowSize)} | ||
t.mu.Lock() | ||
if t.activeStreams != nil { // Can be niled from Close() | ||
t.activeStreams[s.id] = s | ||
} | ||
t.mu.Unlock() | ||
if t.streamQuota > 0 && t.waitingStreams > 0 { | ||
select { | ||
case t.streamsQuotaAvailable <- struct{}{}: | ||
|
@@ -744,13 +748,7 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, | |
} | ||
for { | ||
success, err := t.controlBuf.executeAndPut(func(it interface{}) bool { | ||
if !checkForStreamQuota(it) { | ||
return false | ||
} | ||
if !checkForHeaderListSize(it) { | ||
return false | ||
} | ||
return true | ||
return checkForHeaderListSize(it) && checkForStreamQuota(it) | ||
}, hdr) | ||
if err != nil { | ||
// Connection closed. | ||
|
@@ -1003,13 +1001,13 @@ func (t *http2Client) updateWindow(s *Stream, n uint32) { | |
// for the transport and the stream based on the current bdp | ||
// estimation. | ||
func (t *http2Client) updateFlowControl(n uint32) { | ||
t.mu.Lock() | ||
for _, s := range t.activeStreams { | ||
s.fc.newLimit(n) | ||
} | ||
t.mu.Unlock() | ||
updateIWS := func(interface{}) bool { | ||
t.initialWindowSize = int32(n) | ||
t.mu.Lock() | ||
for _, s := range t.activeStreams { | ||
s.fc.newLimit(n) | ||
} | ||
t.mu.Unlock() | ||
return true | ||
} | ||
t.controlBuf.executeAndPut(updateIWS, &outgoingWindowUpdate{streamID: 0, increment: t.fc.newLimit(n)}) | ||
|
@@ -1215,7 +1213,7 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { | |
default: | ||
t.setGoAwayReason(f) | ||
close(t.goAway) | ||
t.controlBuf.put(&incomingGoAway{}) | ||
defer t.controlBuf.put(&incomingGoAway{}) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please add a comment explaining that we defer because There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Added "Defer as t.mu is currently held". |
||
// Notify the clientconn about the GOAWAY before we set the state to | ||
// draining, to allow the client to stop attempting to create streams | ||
// before disallowing new streams on this connection. | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -8041,3 +8041,45 @@ func (s) TestServerClosesConn(t *testing.T) { | |
} | ||
t.Fatalf("timed out waiting for conns to be closed by server; still open: %v", atomic.LoadInt32(&wrapLis.connsOpen)) | ||
} | ||
|
||
// TestUnexpectedEOF tests a scenario where a client invokes two unary RPC | ||
// calls. The first call receives a payload which exceeds max grpc receive | ||
// message length, and the second gets a large response. This second RPC should | ||
// not fail with unexpected.EOF. | ||
func (s) TestUnexpectedEOF(t *testing.T) { | ||
ss := &stubserver.StubServer{ | ||
UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { | ||
return &testpb.SimpleResponse{ | ||
Payload: &testpb.Payload{ | ||
Body: bytes.Repeat([]byte("a"), int(in.ResponseSize)), | ||
}, | ||
}, nil | ||
}, | ||
} | ||
if err := ss.Start([]grpc.ServerOption{}); err != nil { | ||
t.Fatalf("Error starting endpoint server: %v", err) | ||
} | ||
defer ss.Stop() | ||
|
||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) | ||
defer cancel() | ||
for i := 0; i < 10; i++ { | ||
// exceeds grpc.DefaultMaxRecvMessageSize, this should error with | ||
// RESOURCE_EXHAUSTED error. | ||
_, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{ResponseSize: 4194304}) | ||
if err != nil { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: combine the assignment and the conditional statements. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||
// This check doesn't fail on a non status error. However, the main | ||
// this is testing for is an unexpected EOF with a status code | ||
// INTERNAL so this is fine. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Delete now. It does fail on a non-status error since code would be Unknown in that case. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Whoops forgot deleted. |
||
if code := status.Code(err); code != codes.ResourceExhausted { | ||
t.Fatalf("unexpected err in UnaryCall: %v", err) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could you mention the received and expected codes in the error string. got, want := status.Code(err), codes.ResourceExhausted; got != want {
t.Fatalf("UnaryCall RPC returned status code: %v, want %v", got, want)
} There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Printing the whole error is better than only printing the code, as there might be status message info that's useful: t.Fatalf("UnaryCall RPC returned error: %v, want status code %v", err, want) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Switched to this: |
||
} | ||
} | ||
// Larger response that doesn't exceed DefaultMaxRecvMessageSize, this | ||
// should work normally. | ||
_, err = ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{ResponseSize: 275075}) | ||
if err != nil { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: combine the assignment and the conditional statements. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||
t.Fatalf("unexpected err in UnaryCall: %v", err) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Prefer concise error messages: t.Fatalf("UnaryCall RPC failed: %v", err) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Switched. |
||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Switched.