diff --git a/daemon/graphdriver/aufs/aufs.go b/daemon/graphdriver/aufs/aufs.go index 8ba097d45a507..3d72a8f012c20 100644 --- a/daemon/graphdriver/aufs/aufs.go +++ b/daemon/graphdriver/aufs/aufs.go @@ -300,11 +300,14 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) { // AUFS doesn't need the parent layer to produce a diff. return archive.TarWithOptions(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{ Compression: archive.Uncompressed, + Excludes: []string{".wh..wh.*"}, }) } func (a *Driver) applyDiff(id string, diff archive.ArchiveReader) error { - return archive.Untar(diff, path.Join(a.rootPath(), "diff", id), nil) + return archive.Untar(diff, path.Join(a.rootPath(), "diff", id), &archive.TarOptions{ + Excludes: []string{".wh..wh."}, + }) } // DiffSize calculates the changes between the specified id diff --git a/graph/graph.go b/graph/graph.go index 75b1825034d60..720f6e6963820 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -132,14 +132,14 @@ func (graph *Graph) Create(layerData archive.ArchiveReader, containerID, contain img.ContainerConfig = *containerConfig } - if err := graph.Register(img, nil, layerData); err != nil { + if err := graph.Register(img, layerData); err != nil { return nil, err } return img, nil } // Register imports a pre-existing image into the graph. -func (graph *Graph) Register(img *image.Image, jsonData []byte, layerData archive.ArchiveReader) (err error) { +func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) (err error) { defer func() { // If any error occurs, remove the new dir from the driver. // Don't check for errors since the dir might not have been created. @@ -181,7 +181,7 @@ func (graph *Graph) Register(img *image.Image, jsonData []byte, layerData archiv } // Apply the diff/layer img.SetGraph(graph) - if err := image.StoreImage(img, jsonData, layerData, tmp); err != nil { + if err := image.StoreImage(img, layerData, tmp); err != nil { return err } // Commit diff --git a/graph/load.go b/graph/load.go index 05e963daaa86e..875741ecf7cef 100644 --- a/graph/load.go +++ b/graph/load.go @@ -118,7 +118,7 @@ func (s *TagStore) recursiveLoad(eng *engine.Engine, address, tmpImageDir string } } } - if err := s.graph.Register(img, imageJson, layer); err != nil { + if err := s.graph.Register(img, layer); err != nil { return err } } diff --git a/graph/pull.go b/graph/pull.go index 9345d7d4892fa..cd9168e76976b 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -392,8 +392,7 @@ func (s *TagStore) pullImage(r *registry.Session, out io.Writer, imgID, endpoint layers_downloaded = true defer layer.Close() - err = s.graph.Register(img, imgJSON, - utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading")) + err = s.graph.Register(img, utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading")) if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries { time.Sleep(time.Duration(j) * 500 * time.Millisecond) continue @@ -577,7 +576,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri defer d.tmpFile.Close() d.tmpFile.Seek(0, 0) if d.tmpFile != nil { - err = s.graph.Register(d.img, d.imgJSON, + err = s.graph.Register(d.img, utils.ProgressReader(d.tmpFile, int(d.length), out, sf, false, utils.TruncateID(d.img.ID), "Extracting")) if err != nil { return false, err diff --git a/graph/service.go b/graph/service.go index 9b1509af296bb..04c5e8d455562 100644 --- a/graph/service.go +++ b/graph/service.go @@ -6,12 +6,10 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" - "github.com/docker/docker/image" ) func (s *TagStore) Install(eng *engine.Engine) error { for name, handler := range map[string]engine.Handler{ - "image_set": s.CmdSet, "image_tag": s.CmdTag, "tag": s.CmdTagLegacy, // FIXME merge with "image_tag" "image_get": s.CmdGet, @@ -33,53 +31,6 @@ func (s *TagStore) Install(eng *engine.Engine) error { return nil } -// CmdSet stores a new image in the graph. -// Images are stored in the graph using 4 elements: -// - A user-defined ID -// - A collection of metadata describing the image -// - A directory tree stored as a tar archive (also called the "layer") -// - A reference to a "parent" ID on top of which the layer should be applied -// -// NOTE: even though the parent ID is only useful in relation to the layer and how -// to apply it (ie you could represent the full directory tree as 'parent_layer + layer', -// it is treated as a top-level property of the image. This is an artifact of early -// design and should probably be cleaned up in the future to simplify the design. -// -// Syntax: image_set ID -// Input: -// - Layer content must be streamed in tar format on stdin. An empty input is -// valid and represents a nil layer. -// -// - Image metadata must be passed in the command environment. -// 'json': a json-encoded object with all image metadata. -// It will be stored as-is, without any encoding/decoding artifacts. -// That is a requirement of the current registry client implementation, -// because a re-encoded json might invalidate the image checksum at -// the next upload, even with functionaly identical content. -func (s *TagStore) CmdSet(job *engine.Job) engine.Status { - if len(job.Args) != 1 { - return job.Errorf("usage: %s NAME", job.Name) - } - var ( - imgJSON = []byte(job.Getenv("json")) - layer = job.Stdin - ) - if len(imgJSON) == 0 { - return job.Errorf("mandatory key 'json' is not set") - } - // We have to pass an *image.Image object, even though it will be completely - // ignored in favor of the redundant json data. - // FIXME: the current prototype of Graph.Register is stupid and redundant. - img, err := image.NewImgJSON(imgJSON) - if err != nil { - return job.Error(err) - } - if err := s.graph.Register(img, imgJSON, layer); err != nil { - return job.Error(err) - } - return engine.StatusOK -} - // CmdGet returns information about an image. // If the image doesn't exist, an empty object is returned, to allow // checking for an image's existence. @@ -150,6 +101,7 @@ func (s *TagStore) CmdLookup(job *engine.Job) engine.Status { out.Set("Os", image.OS) out.SetInt64("Size", image.Size) out.SetInt64("VirtualSize", image.GetParentsSize(0)+image.Size) + out.Set("Checksum", image.Checksum) if _, err = out.WriteTo(job.Stdout); err != nil { return job.Error(err) } diff --git a/graph/tags_unit_test.go b/graph/tags_unit_test.go index e4f1fb809fd8b..1b87565dc7261 100644 --- a/graph/tags_unit_test.go +++ b/graph/tags_unit_test.go @@ -62,7 +62,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { t.Fatal(err) } img := &image.Image{ID: testImageID} - if err := graph.Register(img, nil, archive); err != nil { + if err := graph.Register(img, archive); err != nil { t.Fatal(err) } if err := store.Set(testImageName, "", testImageID, false); err != nil { diff --git a/image/image.go b/image/image.go index 728a188a1470b..24c25f3cf13e3 100644 --- a/image/image.go +++ b/image/image.go @@ -11,6 +11,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/runconfig" "github.com/docker/docker/utils" ) @@ -32,6 +33,7 @@ type Image struct { Config *runconfig.Config `json:"config,omitempty"` Architecture string `json:"architecture,omitempty"` OS string `json:"os,omitempty"` + Checksum string `json:"checksum"` Size int64 graph Graph @@ -70,7 +72,7 @@ func LoadImage(root string) (*Image, error) { return img, nil } -func StoreImage(img *Image, jsonData []byte, layerData archive.ArchiveReader, root string) error { +func StoreImage(img *Image, layerData archive.ArchiveReader, root string) error { // Store the layer var ( size int64 @@ -80,9 +82,26 @@ func StoreImage(img *Image, jsonData []byte, layerData archive.ArchiveReader, ro // If layerData is not nil, unpack it into the new layer if layerData != nil { - if size, err = driver.ApplyDiff(img.ID, img.Parent, layerData); err != nil { + // Autodetect compression of the layer data. + layerData, err := archive.DecompressStream(layerData) + if err != nil { + return err + } + defer layerData.Close() + + // Wrap with tarsum. + tarsumLayerData, err := tarsum.NewTarSum(layerData, true, tarsum.VersionDev) + if err != nil { return err } + + // Extract the archive to the file system driver. + if size, err = driver.ApplyDiff(img.ID, img.Parent, tarsumLayerData); err != nil { + return err + } + + // Get the resulting tarsum. + img.Checksum = tarsumLayerData.Sum(nil) } img.Size = size @@ -90,19 +109,16 @@ func StoreImage(img *Image, jsonData []byte, layerData archive.ArchiveReader, ro return err } - // If raw json is provided, then use it - if jsonData != nil { - if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { - return err - } - } else { - if jsonData, err = json.Marshal(img); err != nil { - return err - } - if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { - return err - } + var jsonData []byte + + if jsonData, err = json.Marshal(img); err != nil { + return err } + + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err + } + return nil } diff --git a/integration/graph_test.go b/integration/graph_test.go index 203476cbb2c70..56e5a90642ddb 100644 --- a/integration/graph_test.go +++ b/integration/graph_test.go @@ -74,7 +74,7 @@ func TestInterruptedRegister(t *testing.T) { Created: time.Now(), } w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling) - graph.Register(image, nil, badArchive) + graph.Register(image, badArchive) if _, err := graph.Get(image.ID); err == nil { t.Fatal("Image should not exist after Register is interrupted") } @@ -83,7 +83,7 @@ func TestInterruptedRegister(t *testing.T) { if err != nil { t.Fatal(err) } - if err := graph.Register(image, nil, goodArchive); err != nil { + if err := graph.Register(image, goodArchive); err != nil { t.Fatal(err) } } @@ -133,7 +133,7 @@ func TestRegister(t *testing.T) { Comment: "testing", Created: time.Now(), } - err = graph.Register(image, nil, archive) + err = graph.Register(image, archive) if err != nil { t.Fatal(err) } @@ -228,7 +228,7 @@ func TestDelete(t *testing.T) { t.Fatal(err) } // Test delete twice (pull -> rm -> pull -> rm) - if err := graph.Register(img1, nil, archive); err != nil { + if err := graph.Register(img1, archive); err != nil { t.Fatal(err) } if err := graph.Delete(img1.ID); err != nil { @@ -262,9 +262,9 @@ func TestByParent(t *testing.T) { Created: time.Now(), Parent: parentImage.ID, } - _ = graph.Register(parentImage, nil, archive1) - _ = graph.Register(childImage1, nil, archive2) - _ = graph.Register(childImage2, nil, archive3) + _ = graph.Register(parentImage, archive1) + _ = graph.Register(childImage1, archive2) + _ = graph.Register(childImage2, archive3) byParent, err := graph.ByParent() if err != nil {