Skip to content

Commit

Permalink
Merge pull request #3048 from tonistiigi/v0.10.4-picks
Browse files Browse the repository at this point in the history 
[v0.10] cherry-picks for v0.10.4
  • Loading branch information
@tonistiigi
tonistiigi committed Aug 22, 2022
2 parents 8e2d9b9 + 36e7f50 commit a2ba686
Show file tree
Hide file tree
Showing 27 changed files with 349 additions and 2,288 deletions.
3 changes: 1 addition & 2 deletions executor/oci/resolvconf.go
View file
Expand Up @@ -7,7 +7,6 @@ import (
"path/filepath"

"github.com/docker/docker/libnetwork/resolvconf"
"github.com/docker/docker/libnetwork/types"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/util/flightcontrol"
"github.com/pkg/errors"
Expand Down Expand Up @@ -74,7 +73,7 @@ func GetResolvConf(ctx context.Context, stateDir string, idmap *idtools.Identity

if dns != nil {
var (
dnsNameservers = resolvconf.GetNameservers(dt, types.IP)
dnsNameservers = resolvconf.GetNameservers(dt, resolvconf.IP)
dnsSearchDomains = resolvconf.GetSearchDomains(dt)
dnsOptions = resolvconf.GetOptions(dt)
)
Expand Down
7 changes: 4 additions & 3 deletions frontend/dockerfile/builder/build.go
View file
Expand Up @@ -463,7 +463,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
}
c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url))
},
ContextByName: contextByNameFunc(c, tp),
ContextByName: contextByNameFunc(c),
})

if err != nil {
Expand Down Expand Up @@ -812,8 +812,8 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c
return opts
}

func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string, string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
return func(ctx context.Context, name, resolveMode string) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
func contextByNameFunc(c client.Client) func(context.Context, string, string, *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
return func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, *binfotypes.BuildInfo, error) {
named, err := reference.ParseNormalizedNamed(name)
if err != nil {
return nil, nil, nil, errors.Wrapf(err, "invalid context name %s", name)
Expand Down Expand Up @@ -879,6 +879,7 @@ func contextByName(ctx context.Context, c client.Client, name string, platform *
if err := json.Unmarshal(data, &img); err != nil {
return nil, nil, nil, err
}
img.Created = nil

st := llb.Image(ref, imgOpt...)
st, err = st.WithImageConfig(data)
Expand Down
38 changes: 21 additions & 17 deletions frontend/dockerfile/dockerfile2llb/convert.go
View file
Expand Up @@ -70,16 +70,19 @@ type ConvertOpt struct {
SourceMap *llb.SourceMap
Hostname string
Warn func(short, url string, detail [][]byte, location *parser.Range)
ContextByName func(ctx context.Context, name, resolveMode string) (*llb.State, *Image, *binfotypes.BuildInfo, error)
ContextByName func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *Image, *binfotypes.BuildInfo, error)
}

func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
buildInfo := &binfotypes.BuildInfo{}
contextByName := opt.ContextByName
opt.ContextByName = func(ctx context.Context, name, resolveMode string) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
opt.ContextByName = func(ctx context.Context, name, resolveMode string, p *ocispecs.Platform) (*llb.State, *Image, *binfotypes.BuildInfo, error) {
if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") {
if contextByName != nil {
st, img, bi, err := contextByName(ctx, name, resolveMode)
if p == nil {
p = opt.TargetPlatform
}
st, img, bi, err := contextByName(ctx, name, resolveMode, p)
if err != nil {
return nil, nil, nil, err
}
Expand Down Expand Up @@ -165,8 +168,21 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
prefixPlatform: opt.PrefixPlatform,
}

if v := st.Platform; v != "" {
v, err := shlex.ProcessWordWithMap(v, metaArgsToMap(optMetaArgs))
if err != nil {
return nil, nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to process arguments for platform %s", v), st.Location)
}

p, err := platforms.Parse(v)
if err != nil {
return nil, nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to parse platform %s", v), st.Location)
}
ds.platform = &p
}

if st.Name != "" {
s, img, bi, err := opt.ContextByName(ctx, st.Name, opt.ImageResolveMode.String())
s, img, bi, err := opt.ContextByName(ctx, st.Name, opt.ImageResolveMode.String(), ds.platform)
if err != nil {
return nil, nil, nil, err
}
Expand Down Expand Up @@ -195,18 +211,6 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
ds.stageName = fmt.Sprintf("stage-%d", i)
}

if v := st.Platform; v != "" {
v, err := shlex.ProcessWordWithMap(v, metaArgsToMap(optMetaArgs))
if err != nil {
return nil, nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to process arguments for platform %s", v), st.Location)
}

p, err := platforms.Parse(v)
if err != nil {
return nil, nil, nil, parser.WithLocation(errors.Wrapf(err, "failed to parse platform %s", v), st.Location)
}
ds.platform = &p
}
allDispatchStates.addState(ds)

total := 0
Expand Down Expand Up @@ -313,7 +317,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
d.stage.BaseName = reference.TagNameOnly(ref).String()

var isScratch bool
st, img, bi, err := opt.ContextByName(ctx, d.stage.BaseName, opt.ImageResolveMode.String())
st, img, bi, err := opt.ContextByName(ctx, d.stage.BaseName, opt.ImageResolveMode.String(), platform)
if err != nil {
return err
}
Expand Down
169 changes: 168 additions & 1 deletion frontend/dockerfile/dockerfile_test.go
View file
Expand Up @@ -116,6 +116,8 @@ var allTests = integration.TestFuncs(
testUlimit,
testCgroupParent,
testNamedImageContext,
testNamedImageContextPlatform,
testNamedImageContextTimestamps,
testNamedLocalContext,
testNamedInputContext,
testNamedMultiplatformInputContext,
Expand Down Expand Up @@ -5434,6 +5436,163 @@ COPY --from=base /env_foobar /
require.Contains(t, string(dt), "/foobar:")
}

func testNamedImageContextPlatform(t *testing.T, sb integration.Sandbox) {
ctx := sb.Context()

c, err := client.New(ctx, sb.Address())
require.NoError(t, err)
defer c.Close()

registry, err := sb.NewRegistry()
if errors.Is(err, integration.ErrRequirements) {
t.Skip(err.Error())
}
require.NoError(t, err)

// Build a base image and force buildkit to generate a manifest list.
dockerfile := []byte(`FROM --platform=$BUILDPLATFORM alpine:latest`)
target := registry + "/buildkit/testnamedimagecontextplatform:latest"

dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
)
require.NoError(t, err)

f := getFrontend(t, sb)

_, err = f.Solve(sb.Context(), c, client.SolveOpt{
FrontendAttrs: map[string]string{
"build-arg:BUILDKIT_MULTI_PLATFORM": "true",
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
Exports: []client.ExportEntry{
{
Type: client.ExporterImage,
Attrs: map[string]string{
"name": target,
"push": "true",
},
},
},
}, nil)
require.NoError(t, err)

dockerfile = []byte(`
FROM --platform=$BUILDPLATFORM busybox AS target
RUN echo hello
`)

dir, err = tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
)
require.NoError(t, err)

f = getFrontend(t, sb)

_, err = f.Solve(sb.Context(), c, client.SolveOpt{
FrontendAttrs: map[string]string{
"context:busybox": "docker-image://" + target,
// random platform that would never exist so it doesn't conflict with the build machine
// here we specifically want to make sure that the platform chosen for the image source is the one in the dockerfile not the target platform.
"platform": "darwin/ppc64le",
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
}, nil)
require.NoError(t, err)
}

func testNamedImageContextTimestamps(t *testing.T, sb integration.Sandbox) {
ctx := sb.Context()

c, err := client.New(ctx, sb.Address())
require.NoError(t, err)
defer c.Close()

registry, err := sb.NewRegistry()
if errors.Is(err, integration.ErrRequirements) {
t.Skip(err.Error())
}
require.NoError(t, err)

f := getFrontend(t, sb)

dockerfile := []byte(`
FROM alpine
RUN echo foo >> /test
`)
dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)

target := registry + "/buildkit/testnamedimagecontexttimestamps:latest"
_, err = f.Solve(sb.Context(), c, client.SolveOpt{
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dir,
builder.DefaultLocalNameContext: dir,
},
Exports: []client.ExportEntry{
{
Type: client.ExporterImage,
Attrs: map[string]string{
"name": target,
"push": "true",
},
},
},
}, nil)
require.NoError(t, err)

desc, provider, err := contentutil.ProviderFromRef(target)
require.NoError(t, err)
img, err := readImage(sb.Context(), provider, desc)
require.NoError(t, err)

dirDerived, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dirDerived)

targetDerived := registry + "/buildkit/testnamedimagecontexttimestampsderived:latest"
_, err = f.Solve(sb.Context(), c, client.SolveOpt{
FrontendAttrs: map[string]string{
"context:alpine": "docker-image://" + target,
},
LocalDirs: map[string]string{
builder.DefaultLocalNameDockerfile: dirDerived,
builder.DefaultLocalNameContext: dirDerived,
},
Exports: []client.ExportEntry{
{
Type: client.ExporterImage,
Attrs: map[string]string{
"name": targetDerived,
"push": "true",
},
},
},
}, nil)
require.NoError(t, err)

desc, provider, err = contentutil.ProviderFromRef(targetDerived)
require.NoError(t, err)
imgDerived, err := readImage(sb.Context(), provider, desc)
require.NoError(t, err)

require.NotEqual(t, img.img.Created, imgDerived.img.Created)
diff := imgDerived.img.Created.Sub(*img.img.Created)
require.Greater(t, diff, time.Duration(0))
require.Less(t, diff, 10*time.Minute)
}

func testNamedLocalContext(t *testing.T, sb integration.Sandbox) {
ctx := sb.Context()

Expand Down Expand Up @@ -6023,6 +6182,7 @@ func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser

type imageInfo struct {
desc ocispecs.Descriptor
img ocispecs.Image
layers []map[string]*testutil.TarItem
}

Expand Down Expand Up @@ -6054,12 +6214,19 @@ func readImage(ctx context.Context, p content.Provider, desc ocispecs.Descriptor
if err != nil {
return nil, err
}

var mfst ocispecs.Manifest
if err := json.Unmarshal(dt, &mfst); err != nil {
return nil, err
}

dt, err = content.ReadBlob(ctx, p, mfst.Config)
if err != nil {
return nil, err
}
if err := json.Unmarshal(dt, &ii.img); err != nil {
return nil, err
}

for _, l := range mfst.Layers {
dt, err := content.ReadBlob(ctx, p, l)
if err != nil {
Expand Down
3 changes: 3 additions & 0 deletions frontend/gateway/container.go
View file
Expand Up @@ -361,6 +361,8 @@ func (gwCtr *gatewayContainer) Start(ctx context.Context, req client.StartReques
}

func (gwCtr *gatewayContainer) Release(ctx context.Context) error {
gwCtr.mu.Lock()
defer gwCtr.mu.Unlock()
gwCtr.cancel()
err1 := gwCtr.errGroup.Wait()

Expand All @@ -371,6 +373,7 @@ func (gwCtr *gatewayContainer) Release(ctx context.Context) error {
err2 = err
}
}
gwCtr.cleanup = nil

if err1 != nil {
return stack.Enable(err1)
Expand Down
6 changes: 6 additions & 0 deletions frontend/gateway/forwarder/forward.go
View file
Expand Up @@ -52,6 +52,7 @@ type bridgeClient struct {
workers worker.Infos
workerRefByID map[string]*worker.WorkerRef
buildOpts client.BuildOpts
ctrs []client.Container
}

func (c *bridgeClient) Solve(ctx context.Context, req client.SolveRequest) (*client.Result, error) {
Expand Down Expand Up @@ -212,6 +213,10 @@ func (c *bridgeClient) toFrontendResult(r *client.Result) (*frontend.Result, err
}

func (c *bridgeClient) discard(err error) {
for _, ctr := range c.ctrs {
ctr.Release(context.TODO())
}

for id, workerRef := range c.workerRefByID {
workerRef.ImmutableRef.Release(context.TODO())
delete(c.workerRefByID, id)
Expand Down Expand Up @@ -300,6 +305,7 @@ func (c *bridgeClient) NewContainer(ctx context.Context, req client.NewContainer
if err != nil {
return nil, err
}
c.ctrs = append(c.ctrs, ctr)
return ctr, nil
}

Expand Down
9 changes: 8 additions & 1 deletion frontend/gateway/gateway.go
View file
Expand Up @@ -278,7 +278,7 @@ func (gf *gatewayFrontend) Solve(ctx context.Context, llbBridge frontend.Fronten
err = w.Executor().Run(ctx, "", mountWithSession(rootFS, session.NewGroup(sid)), mnts, executor.ProcessInfo{Meta: meta, Stdin: lbf.Stdin, Stdout: lbf.Stdout, Stderr: os.Stderr}, nil)

if err != nil {
if errdefs.IsCanceled(err) && lbf.isErrServerClosed {
if errdefs.IsCanceled(ctx, err) && lbf.isErrServerClosed {
err = errors.Errorf("frontend grpc server closed unexpectedly")
}
// An existing error (set via Return rpc) takes
Expand Down Expand Up @@ -345,6 +345,13 @@ func (b *bindMount) IdentityMapping() *idtools.IdentityMapping {
func (lbf *llbBridgeForwarder) Discard() {
lbf.mu.Lock()
defer lbf.mu.Unlock()

for ctr := range lbf.ctrs {
lbf.ReleaseContainer(context.TODO(), &pb.ReleaseContainerRequest{
ContainerID: ctr,
})
}

for id, workerRef := range lbf.workerRefByID {
workerRef.ImmutableRef.Release(context.TODO())
delete(lbf.workerRefByID, id)
Expand Down

0 comments on commit a2ba686

Please sign in to comment.