ADD enhancement: allow adding from another image

docs/sources/reference/builder.rst

@@ -326,6 +326,35 @@ The copy obeys the following rules:
   written at ``<dest>``.
 * If ``<dest>`` doesn't exist, it is created along with all missing
   directories in its path.
+* If ``<src>`` is a URI with the scheme prefix of ``image://``, then followed
+  by the image name (including registry and/or tag, if needed), and has a
+  suffix of ``:`` and the path to be copied from. (i.e.
+  image://[registry[:port]]<name>[:tag]:<path> )
+
+A couple of examples on the usage of this ``image://`` in ``<src>``
+
+.. code-block:: bash
+
+    FROM scratch
+    ADD image://vbatts/fat-buildtime:/build/runtime/ /
+    ENTRYPOINT ["/runtime/bin/myapp"]
+
+Will copy the entire ``/build/runtime/`` directory, from the image
+``vbatts/fat-build``, to the root filesystem of the image being built.
+
+.. note::
+    If you build such a thin runtime, that does not provide ``/bin/sh``, then
+    the ``CMD`` instruction will fail
+
+.. code-block:: bash
+
+    FROM busybox
+    ADD image://my.registry.lan:5000/vbatts/fat-buildtime:stable:/build/runtime/bin/myapp /
+    CMD "/myapp"
+
+Will copy the file ``/build/runtime/bin/myapp`` from
+``my.registry.lan:5000/vbatts/fat-buildtime:stable`` image, to the root
+filesystem of the image being built.
 
 .. _dockerfile_entrypoint:
 

server/buildfile.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/dotcloud/docker/archive"
+	"github.com/dotcloud/docker/image"
 	"github.com/dotcloud/docker/nat"
 	"github.com/dotcloud/docker/registry"
 	"github.com/dotcloud/docker/runconfig"
@@ -74,30 +75,34 @@ func (b *buildFile) clearTmp(containers map[string]struct{}) {
 	}
 }
 
+func (b *buildFile) pullImage(name string) (*image.Image, error) {
+	remote, tag := utils.ParseRepositoryTag(name)
+	pullRegistryAuth := b.authConfig
+	if len(b.configFile.Configs) > 0 {
+		// The request came with a full auth config file, we prefer to use that
+		endpoint, _, err := registry.ResolveRepositoryName(remote)
+		if err != nil {
+			return nil, err
+		}
+		resolvedAuth := b.configFile.ResolveAuthConfig(endpoint)
+		pullRegistryAuth = &resolvedAuth
+	}
+	job := b.srv.Eng.Job("pull", remote, tag)
+	job.SetenvBool("json", b.sf.Json())
+	job.SetenvBool("parallel", true)
+	job.SetenvJson("authConfig", pullRegistryAuth)
+	job.Stdout.Add(b.outOld)
+	if err := job.Run(); err != nil {
+		return nil, err
+	}
+	return b.runtime.Repositories().LookupImage(name)
+}
+
 func (b *buildFile) CmdFrom(name string) error {
 	image, err := b.runtime.Repositories().LookupImage(name)
 	if err != nil {
 		if b.runtime.Graph().IsNotExist(err) {
-			remote, tag := utils.ParseRepositoryTag(name)
-			pullRegistryAuth := b.authConfig
-			if len(b.configFile.Configs) > 0 {
-				// The request came with a full auth config file, we prefer to use that
-				endpoint, _, err := registry.ResolveRepositoryName(remote)
-				if err != nil {
-					return err
-				}
-				resolvedAuth := b.configFile.ResolveAuthConfig(endpoint)
-				pullRegistryAuth = &resolvedAuth
-			}
-			job := b.srv.Eng.Job("pull", remote, tag)
-			job.SetenvBool("json", b.sf.Json())
-			job.SetenvBool("parallel", true)
-			job.SetenvJson("authConfig", pullRegistryAuth)
-			job.Stdout.Add(b.outOld)
-			if err := job.Run(); err != nil {
-				return err
-			}
-			image, err = b.runtime.Repositories().LookupImage(name)
+			image, err = b.pullImage(name)
 			if err != nil {
 				return err
 			}
@@ -476,6 +481,78 @@ func (b *buildFile) CmdAdd(args string) error {
 		isRemote   bool
 	)
 
+	// ADD a path from an existing image, to the new build
+	if utils.IsIMAGE(orig) {
+		// 1) derive the host we are copying files from
+		// 2) does the image exist locally, if not maybe 'pull' it
+		// 3) mount up that image
+		// 4) tar up the src
+		// 4) copy files over to the new
+		srcInfo, err := utils.ParseImageURI(orig)
+		if err != nil {
+			return err
+		}
+		// Check if the image exists
+		if _, err = b.runtime.Repositories().LookupImage(srcInfo["name"]); err != nil {
+			// the error is something besides the image not being present
+			if !b.runtime.Graph().IsNotExist(err) {
+				return err
+			}
+			// otherwise, pull the image
+			_, err = b.pullImage(srcInfo["name"])
+			if err != nil {
+				return err
+			}
+		}
+
+		srcConfig := &runconfig.Config{
+			Image: srcInfo["name"],
+			Cmd:   []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)},
+		}
+		srcContainer, _, err := b.runtime.Create(srcConfig, "")
+		if err != nil {
+			return err
+		}
+		defer b.runtime.Destroy(srcContainer)
+
+		if err := srcContainer.Mount(); err != nil {
+			return err
+		}
+		defer srcContainer.Unmount()
+		srcPath := path.Join(srcContainer.RootfsPath(), srcInfo["path"])
+		srcTar, err := archive.Tar(srcPath, archive.Uncompressed)
+		if err != nil {
+			return err
+		}
+		tarSum := &utils.TarSum{Reader: srcTar, DisableCompression: true}
+
+		tmpDirName, err := ioutil.TempDir(b.contextPath, "docker-add")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(tmpDirName)
+
+		tmpFileName := path.Join(tmpDirName, "from.tar")
+		tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
+		if err != nil {
+			return err
+		}
+		if _, err = io.Copy(tmpFile, tarSum); err != nil {
+			tmpFile.Close()
+			return err
+		}
+		remoteHash = tarSum.Sum(nil)
+		srcTar.Close()
+		origPath = path.Join(filepath.Base(tmpDirName), filepath.Base(tmpFileName))
+
+		// If the destination is a directory, figure out the filename.
+		if strings.HasSuffix(dest, "/") {
+			if destPath, err = b.determineDest(orig, dest); err != nil {
+				return err
+			}
+		}
+	}
+
 	if utils.IsURL(orig) {
 		isRemote = true
 		resp, err := utils.Download(orig)
@@ -510,20 +587,9 @@ func (b *buildFile) CmdAdd(args string) error {
 
 		// If the destination is a directory, figure out the filename.
 		if strings.HasSuffix(dest, "/") {
-			u, err := url.Parse(orig)
-			if err != nil {
+			if destPath, err = b.determineDest(orig, dest); err != nil {
 				return err
 			}
-			path := u.Path
-			if strings.HasSuffix(path, "/") {
-				path = path[:len(path)-1]
-			}
-			parts := strings.Split(path, "/")
-			filename := parts[len(parts)-1]
-			if filename == "" {
-				return fmt.Errorf("cannot determine filename from url: %s", u)
-			}
-			destPath = dest + filename
 		}
 	}
 
@@ -598,6 +664,23 @@ func (b *buildFile) CmdAdd(args string) error {
 	return nil
 }
 
+func (b *buildFile) determineDest(orig, dest string) (string, error) {
+	u, err := url.Parse(orig)
+	if err != nil {
+		return "", err
+	}
+	path := u.Path
+	if strings.HasSuffix(path, "/") {
+		path = path[:len(path)-1]
+	}
+	parts := strings.Split(path, "/")
+	filename := parts[len(parts)-1]
+	if filename == "" {
+		return "", fmt.Errorf("cannot determine filename from url: %s", u)
+	}
+	return dest + filename, nil
+}
+
 func (b *buildFile) create() (*runtime.Container, error) {
 	if b.image == "" {
 		return nil, fmt.Errorf("Please provide a source image with `from` prior to run")

utils/utils.go

@@ -709,6 +709,35 @@ func NewHTTPRequestError(msg string, res *http.Response) error {
 	}
 }
 
+// hardly keeping with RFC3986, but the docker ns/name:tag layout with a
+// "image://" prefix, and :<path> suffix. Like:
+//    image://[registry[:port]]<name>[:tag]:<path>
+func ParseImageURI(str string) (map[string]string, error) {
+	values := make(map[string]string)
+
+	if !IsIMAGE(str) || strings.Count(str, ":") < 2 {
+		return values, ErrImageUriFormat
+	}
+
+	base_str := strings.TrimPrefix(str, "image://")
+	i := strings.LastIndex(base_str, ":")
+	values["name"] = base_str[:i]
+	if len(values["name"]) == 0 {
+		return values, ErrImageUriFormat
+	}
+	values["path"] = base_str[i+1:]
+	if len(values["path"]) == 0 {
+		values["path"] = "/"
+	}
+	return values, nil
+}
+
+var ErrImageUriFormat = errors.New("image:// URI not properly formatted (image://[registry[:port]]<name>[:tag]:<path>)")
+
+func IsIMAGE(str string) bool {
+	return strings.HasPrefix(str, "image://")
+}
+
 func IsURL(str string) bool {
 	return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://")
 }

utils/utils_test.go

@@ -625,3 +625,71 @@ func TestReadSymlinkedDirectoryToFile(t *testing.T) {
 		t.Errorf("failed to remove symlink: %s", err)
 	}
 }
+
+func TestADDpaths(t *testing.T) {
+	var set = []pathCheck{
+		{"http://foo.com/image", true, IsURL},
+		{"https://foo.com/image.tar", true, IsURL},
+		{"foo.com/image.tar", false, IsURL},
+
+		{"git://github.com/dotcloud/docker", true, IsGIT},
+		{"github.com/dotcloud/docker", true, IsGIT},
+		{"https://foo.com/foo.git", true, IsGIT},
+		{"foo.com/repo.git", false, IsGIT},
+		{"git@github.com:user/repo", true, IsGIT},
+		{"git@github.com:user/repo.git", true, IsGIT},
+		{"git@github.com/user/repo", false, IsGIT},
+		{"git@github.com/user/repo.git", false, IsGIT},
+		{"git@foo.com/repo", false, IsGIT},
+
+		{"image://namespace/name:/build/file", true, IsIMAGE},
+		{"image://name:/build/file", true, IsIMAGE},
+		{"image://registry.com/namespace/name:/build/file", true, IsIMAGE},
+		{"image://registry.com:5000/namespace/name:/build/file", true, IsIMAGE},
+		{"image://registry.com:5000/namespace/name:/build/dir/", true, IsIMAGE},
+		{"namespace/name:/build/file", false, IsIMAGE},
+	}
+
+	for _, item := range set {
+		if item.meth(item.value) != item.result {
+			t.Errorf("%s was supposed to return %q, but did not", item.value, item.result)
+		}
+	}
+}
+
+type pathCheck struct {
+	value  string
+	result bool
+	meth   func(str string) bool
+}
+
+func TestImageURI(t *testing.T) {
+	var set = []struct {
+		value    string
+		result   map[string]string
+		hasError bool
+	}{
+		{"image://namespace/name:/build/file", map[string]string{"name": "namespace/name", "path": "/build/file"}, false},
+		{"image://name:/build/file", map[string]string{"name": "name", "path": "/build/file"}, false},
+		{"image://registry.com/namespace/name:/build/file", map[string]string{"name": "registry.com/namespace/name", "path": "/build/file"}, false},
+		{"image://registry.com:5000/namespace/name:/build/file", map[string]string{"name": "registry.com:5000/namespace/name", "path": "/build/file"}, false},
+		{"image://registry.com:5000/namespace/name:/build/dir/", map[string]string{"name": "registry.com:5000/namespace/name", "path": "/build/dir/"}, false},
+		{"namespace/name:/build/file", map[string]string{}, true},
+	}
+
+	for _, item := range set {
+		r, err := ParseImageURI(item.value)
+		if err == nil && item.hasError {
+			t.Errorf("%s should have errored", item.value)
+			continue
+		} else if err != nil && !item.hasError {
+			t.Errorf("%s should _not_ have errored: %s", item.value, err)
+			continue
+		} else if err != nil && item.hasError {
+			continue
+		}
+		if r["name"] != item.result["name"] || r["path"] != item.result["path"] {
+			t.Errorf("%q should match %q", r, item.result)
+		}
+	}
+}