Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix cancellation error not being detected and erroneously cached #2926

Merged
merged 1 commit into from Jul 21, 2022
Merged

fix cancellation error not being detected and erroneously cached #2926

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion frontend/gateway/gateway.go
View file
Open in desktop
Expand Up @@ -278,7 +278,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
err = w.Executor().Run(ctx, "", mountWithSession(rootFS, session.NewGroup(sid)), mnts, executor.ProcessInfo{Meta: meta, Stdin: lbf.Stdin, Stdout: lbf.Stdout, Stderr: os.Stderr}, nil)

if err != nil {
if errdefs.IsCanceled(err) && lbf.isErrServerClosed {
if errdefs.IsCanceled(ctx, err) && lbf.isErrServerClosed {
err = errors.Errorf("frontend grpc server closed unexpectedly")
}
// An existing error (set via Return rpc) takes
Expand Down
18 changes: 16 additions & 2 deletions solver/errdefs/context.go
View file
Open in desktop
Expand Up @@ -3,11 +3,25 @@ package errdefs
import (
"context"
"errors"
"strings"

"github.com/moby/buildkit/util/grpcerrors"
"google.golang.org/grpc/codes"
)

func IsCanceled(err error) bool {
return errors.Is(err, context.Canceled) || grpcerrors.Code(err) == codes.Canceled
func IsCanceled(ctx context.Context, err error) bool {
if errors.Is(err, context.Canceled) || grpcerrors.Code(err) == codes.Canceled {
return true
}
// grpc does not set cancel correctly when stream gets cancelled and then Recv is called
if err != nil && ctx.Err() == context.Canceled {
// when this error comes from containerd it is not typed at all, just concatenated string
if strings.Contains(err.Error(), "EOF") {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn’t seem robust.
Where is the source of this EOF

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At least one of the places should be https://github.com/containerd/containerd/blob/main/content/proxy/content_writer.go#L44 but there might be more.

Should we ever match an error that is not a real cancellation error here it isn't a big issue. We have already verified that our own context was canceled so at least it was the user's intent. Matching non-cancel error in here means that the error does not get cached as we don't consider it a real error and another parallel process requesting the same result would need to try to recompute the result again.

return true
}
if strings.Contains(err.Error(), context.Canceled.Error()) {
return true
}
}
return false
}
7 changes: 3 additions & 4 deletions solver/jobs.go
View file
Open in desktop
Expand Up @@ -3,7 +3,6 @@ package solver
import (
"context"
"fmt"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -704,7 +703,7 @@ func (s *sharedOp) CalcSlowCache(ctx context.Context, index Index, p PreprocessF
if err != nil {
select {
case <-ctx.Done():
if strings.Contains(err.Error(), context.Canceled.Error()) {
if errdefs.IsCanceled(ctx, err) {
complete = false
releaseError(err)
err = errors.Wrap(ctx.Err(), err.Error())
Expand Down Expand Up @@ -770,7 +769,7 @@ func (s *sharedOp) CacheMap(ctx context.Context, index int) (resp *cacheMapResp,
if err != nil {
select {
case <-ctx.Done():
if strings.Contains(err.Error(), context.Canceled.Error()) {
if errdefs.IsCanceled(ctx, err) {
complete = false
releaseError(err)
err = errors.Wrap(ctx.Err(), err.Error())
Expand Down Expand Up @@ -846,7 +845,7 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result,
if err != nil {
select {
case <-ctx.Done():
if strings.Contains(err.Error(), context.Canceled.Error()) {
if errdefs.IsCanceled(ctx, err) {
complete = false
releaseError(err)
err = errors.Wrap(ctx.Err(), err.Error())
Expand Down
3 changes: 1 addition & 2 deletions solver/llbsolver/bridge.go
View file
Open in desktop
Expand Up @@ -3,7 +3,6 @@ package llbsolver
import (
"context"
"fmt"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -290,7 +289,7 @@ func (rp *resultProxy) Result(ctx context.Context) (res solver.CachedResult, err
if err != nil {
select {
case <-ctx.Done():
if strings.Contains(err.Error(), context.Canceled.Error()) {
if errdefs.IsCanceled(ctx, err) {
return v, err
}
default:
Expand Down
2 changes: 1 addition & 1 deletion source/containerimage/pull.go
View file
Open in desktop
Expand Up @@ -272,7 +272,7 @@ func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cach
return nil, p.cacheKeyErr
}
defer func() {
if !errdefs.IsCanceled(err) {
if !errdefs.IsCanceled(ctx, err) {
p.cacheKeyErr = err
}
}()
Expand Down