From c8944bf6f4d7b8dc84d215e547bdc61cc757b8bd Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Thu, 25 Sep 2014 15:55:05 -0700 Subject: [PATCH 1/4] Compute TarSum of image layer data when installing Adds a new `Checksum` field to the current image format to store the result of the tarsum of the file system changes. Signed-off-by: Josh Hawn (github: jlhawn) --- image/image.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/image/image.go b/image/image.go index 728a188a1470b..8171fe4989125 100644 --- a/image/image.go +++ b/image/image.go @@ -11,6 +11,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/runconfig" "github.com/docker/docker/utils" ) @@ -32,6 +33,7 @@ type Image struct { Config *runconfig.Config `json:"config,omitempty"` Architecture string `json:"architecture,omitempty"` OS string `json:"os,omitempty"` + Checksum string `json:"checksum"` Size int64 graph Graph @@ -80,9 +82,25 @@ func StoreImage(img *Image, jsonData []byte, layerData archive.ArchiveReader, ro // If layerData is not nil, unpack it into the new layer if layerData != nil { - if size, err = driver.ApplyDiff(img.ID, img.Parent, layerData); err != nil { + // Autodetect compression of the layer data. + layerData, err := archive.DecompressStream(layerData) + if err != nil { + return err + } + + // Wrap with tarsum. + tarsumLayerData, err := tarsum.NewTarSum(layerData, true, tarsum.Version0) + if err != nil { return err } + + // Extract the archive to the file system driver. + if size, err = driver.ApplyDiff(img.ID, img.Parent, tarsumLayerData); err != nil { + return err + } + + // Get the resulting tarsum. + img.Checksum = tarsumLayerData.Sum(nil) } img.Size = size From 70e8ba6b9f55075aa082c3dee5a33c5d5fead28c Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Thu, 25 Sep 2014 18:23:42 -0700 Subject: [PATCH 2/4] Remove `jsonData` argument from `image.StoreImage` The argument specified the json data to save to disk when registering a new image into the image graph. If it is nil, then the given image is serialized to json and that is written by default. This default behavior is sufficient because the given image was originally deserialzed from this jsonData to begin with. Signed-off-by: Josh Hawn (github: jlhawn) --- graph/graph.go | 6 +++--- graph/load.go | 2 +- graph/pull.go | 3 +-- graph/service.go | 2 +- graph/tags_unit_test.go | 2 +- image/image.go | 23 ++++++++++------------- integration/graph_test.go | 14 +++++++------- 7 files changed, 24 insertions(+), 28 deletions(-) diff --git a/graph/graph.go b/graph/graph.go index 75b1825034d60..720f6e6963820 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -132,14 +132,14 @@ func (graph *Graph) Create(layerData archive.ArchiveReader, containerID, contain img.ContainerConfig = *containerConfig } - if err := graph.Register(img, nil, layerData); err != nil { + if err := graph.Register(img, layerData); err != nil { return nil, err } return img, nil } // Register imports a pre-existing image into the graph. -func (graph *Graph) Register(img *image.Image, jsonData []byte, layerData archive.ArchiveReader) (err error) { +func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) (err error) { defer func() { // If any error occurs, remove the new dir from the driver. // Don't check for errors since the dir might not have been created. @@ -181,7 +181,7 @@ func (graph *Graph) Register(img *image.Image, jsonData []byte, layerData archiv } // Apply the diff/layer img.SetGraph(graph) - if err := image.StoreImage(img, jsonData, layerData, tmp); err != nil { + if err := image.StoreImage(img, layerData, tmp); err != nil { return err } // Commit diff --git a/graph/load.go b/graph/load.go index 05e963daaa86e..875741ecf7cef 100644 --- a/graph/load.go +++ b/graph/load.go @@ -118,7 +118,7 @@ func (s *TagStore) recursiveLoad(eng *engine.Engine, address, tmpImageDir string } } } - if err := s.graph.Register(img, imageJson, layer); err != nil { + if err := s.graph.Register(img, layer); err != nil { return err } } diff --git a/graph/pull.go b/graph/pull.go index 9345d7d4892fa..00b840b2e7ba4 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -392,8 +392,7 @@ func (s *TagStore) pullImage(r *registry.Session, out io.Writer, imgID, endpoint layers_downloaded = true defer layer.Close() - err = s.graph.Register(img, imgJSON, - utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading")) + err = s.graph.Register(img, utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading")) if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries { time.Sleep(time.Duration(j) * 500 * time.Millisecond) continue diff --git a/graph/service.go b/graph/service.go index 9b1509af296bb..6f020e8d02d3c 100644 --- a/graph/service.go +++ b/graph/service.go @@ -74,7 +74,7 @@ func (s *TagStore) CmdSet(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - if err := s.graph.Register(img, imgJSON, layer); err != nil { + if err := s.graph.Register(img, layer); err != nil { return job.Error(err) } return engine.StatusOK diff --git a/graph/tags_unit_test.go b/graph/tags_unit_test.go index e4f1fb809fd8b..1b87565dc7261 100644 --- a/graph/tags_unit_test.go +++ b/graph/tags_unit_test.go @@ -62,7 +62,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { t.Fatal(err) } img := &image.Image{ID: testImageID} - if err := graph.Register(img, nil, archive); err != nil { + if err := graph.Register(img, archive); err != nil { t.Fatal(err) } if err := store.Set(testImageName, "", testImageID, false); err != nil { diff --git a/image/image.go b/image/image.go index 8171fe4989125..0312012c0a675 100644 --- a/image/image.go +++ b/image/image.go @@ -72,7 +72,7 @@ func LoadImage(root string) (*Image, error) { return img, nil } -func StoreImage(img *Image, jsonData []byte, layerData archive.ArchiveReader, root string) error { +func StoreImage(img *Image, layerData archive.ArchiveReader, root string) error { // Store the layer var ( size int64 @@ -108,19 +108,16 @@ func StoreImage(img *Image, jsonData []byte, layerData archive.ArchiveReader, ro return err } - // If raw json is provided, then use it - if jsonData != nil { - if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { - return err - } - } else { - if jsonData, err = json.Marshal(img); err != nil { - return err - } - if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { - return err - } + var jsonData []byte + + if jsonData, err = json.Marshal(img); err != nil { + return err + } + + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err } + return nil } diff --git a/integration/graph_test.go b/integration/graph_test.go index 203476cbb2c70..56e5a90642ddb 100644 --- a/integration/graph_test.go +++ b/integration/graph_test.go @@ -74,7 +74,7 @@ func TestInterruptedRegister(t *testing.T) { Created: time.Now(), } w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling) - graph.Register(image, nil, badArchive) + graph.Register(image, badArchive) if _, err := graph.Get(image.ID); err == nil { t.Fatal("Image should not exist after Register is interrupted") } @@ -83,7 +83,7 @@ func TestInterruptedRegister(t *testing.T) { if err != nil { t.Fatal(err) } - if err := graph.Register(image, nil, goodArchive); err != nil { + if err := graph.Register(image, goodArchive); err != nil { t.Fatal(err) } } @@ -133,7 +133,7 @@ func TestRegister(t *testing.T) { Comment: "testing", Created: time.Now(), } - err = graph.Register(image, nil, archive) + err = graph.Register(image, archive) if err != nil { t.Fatal(err) } @@ -228,7 +228,7 @@ func TestDelete(t *testing.T) { t.Fatal(err) } // Test delete twice (pull -> rm -> pull -> rm) - if err := graph.Register(img1, nil, archive); err != nil { + if err := graph.Register(img1, archive); err != nil { t.Fatal(err) } if err := graph.Delete(img1.ID); err != nil { @@ -262,9 +262,9 @@ func TestByParent(t *testing.T) { Created: time.Now(), Parent: parentImage.ID, } - _ = graph.Register(parentImage, nil, archive1) - _ = graph.Register(childImage1, nil, archive2) - _ = graph.Register(childImage2, nil, archive3) + _ = graph.Register(parentImage, archive1) + _ = graph.Register(childImage1, archive2) + _ = graph.Register(childImage2, archive3) byParent, err := graph.ByParent() if err != nil { From e37eba20526b5d335007c2970bf894578a956eb9 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Thu, 25 Sep 2014 18:28:52 -0700 Subject: [PATCH 3/4] Remove unused "image_set" job A quick grep of the code base shows that the "image_set" job registered by the TagStore is not used. Signed-off-by: Josh Hawn (github: jlhawn) --- graph/service.go | 49 ------------------------------------------------ 1 file changed, 49 deletions(-) diff --git a/graph/service.go b/graph/service.go index 6f020e8d02d3c..f8c548ae3cebe 100644 --- a/graph/service.go +++ b/graph/service.go @@ -6,12 +6,10 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" - "github.com/docker/docker/image" ) func (s *TagStore) Install(eng *engine.Engine) error { for name, handler := range map[string]engine.Handler{ - "image_set": s.CmdSet, "image_tag": s.CmdTag, "tag": s.CmdTagLegacy, // FIXME merge with "image_tag" "image_get": s.CmdGet, @@ -33,53 +31,6 @@ func (s *TagStore) Install(eng *engine.Engine) error { return nil } -// CmdSet stores a new image in the graph. -// Images are stored in the graph using 4 elements: -// - A user-defined ID -// - A collection of metadata describing the image -// - A directory tree stored as a tar archive (also called the "layer") -// - A reference to a "parent" ID on top of which the layer should be applied -// -// NOTE: even though the parent ID is only useful in relation to the layer and how -// to apply it (ie you could represent the full directory tree as 'parent_layer + layer', -// it is treated as a top-level property of the image. This is an artifact of early -// design and should probably be cleaned up in the future to simplify the design. -// -// Syntax: image_set ID -// Input: -// - Layer content must be streamed in tar format on stdin. An empty input is -// valid and represents a nil layer. -// -// - Image metadata must be passed in the command environment. -// 'json': a json-encoded object with all image metadata. -// It will be stored as-is, without any encoding/decoding artifacts. -// That is a requirement of the current registry client implementation, -// because a re-encoded json might invalidate the image checksum at -// the next upload, even with functionaly identical content. -func (s *TagStore) CmdSet(job *engine.Job) engine.Status { - if len(job.Args) != 1 { - return job.Errorf("usage: %s NAME", job.Name) - } - var ( - imgJSON = []byte(job.Getenv("json")) - layer = job.Stdin - ) - if len(imgJSON) == 0 { - return job.Errorf("mandatory key 'json' is not set") - } - // We have to pass an *image.Image object, even though it will be completely - // ignored in favor of the redundant json data. - // FIXME: the current prototype of Graph.Register is stupid and redundant. - img, err := image.NewImgJSON(imgJSON) - if err != nil { - return job.Error(err) - } - if err := s.graph.Register(img, layer); err != nil { - return job.Error(err) - } - return engine.StatusOK -} - // CmdGet returns information about an image. // If the image doesn't exist, an empty object is returned, to allow // checking for an image's existence. From 4d6216cef9d6f72bc4cf6e474b540f98c3093199 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Mon, 20 Oct 2014 11:42:13 -0700 Subject: [PATCH 4/4] AUFS Diff/ApplyDiff excludes /.wh..wh.* files Also includes Checksum field in `docker inspect` output for images. Correcly close layerData decompress/reader wrapper when storing images. Remove imgJSON from call to graph.Register when pulling an image. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- daemon/graphdriver/aufs/aufs.go | 5 ++++- graph/pull.go | 2 +- graph/service.go | 1 + image/image.go | 3 ++- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/daemon/graphdriver/aufs/aufs.go b/daemon/graphdriver/aufs/aufs.go index 8ba097d45a507..3d72a8f012c20 100644 --- a/daemon/graphdriver/aufs/aufs.go +++ b/daemon/graphdriver/aufs/aufs.go @@ -300,11 +300,14 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) { // AUFS doesn't need the parent layer to produce a diff. return archive.TarWithOptions(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{ Compression: archive.Uncompressed, + Excludes: []string{".wh..wh.*"}, }) } func (a *Driver) applyDiff(id string, diff archive.ArchiveReader) error { - return archive.Untar(diff, path.Join(a.rootPath(), "diff", id), nil) + return archive.Untar(diff, path.Join(a.rootPath(), "diff", id), &archive.TarOptions{ + Excludes: []string{".wh..wh."}, + }) } // DiffSize calculates the changes between the specified id diff --git a/graph/pull.go b/graph/pull.go index 00b840b2e7ba4..cd9168e76976b 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -576,7 +576,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri defer d.tmpFile.Close() d.tmpFile.Seek(0, 0) if d.tmpFile != nil { - err = s.graph.Register(d.img, d.imgJSON, + err = s.graph.Register(d.img, utils.ProgressReader(d.tmpFile, int(d.length), out, sf, false, utils.TruncateID(d.img.ID), "Extracting")) if err != nil { return false, err diff --git a/graph/service.go b/graph/service.go index f8c548ae3cebe..04c5e8d455562 100644 --- a/graph/service.go +++ b/graph/service.go @@ -101,6 +101,7 @@ func (s *TagStore) CmdLookup(job *engine.Job) engine.Status { out.Set("Os", image.OS) out.SetInt64("Size", image.Size) out.SetInt64("VirtualSize", image.GetParentsSize(0)+image.Size) + out.Set("Checksum", image.Checksum) if _, err = out.WriteTo(job.Stdout); err != nil { return job.Error(err) } diff --git a/image/image.go b/image/image.go index 0312012c0a675..24c25f3cf13e3 100644 --- a/image/image.go +++ b/image/image.go @@ -87,9 +87,10 @@ func StoreImage(img *Image, layerData archive.ArchiveReader, root string) error if err != nil { return err } + defer layerData.Close() // Wrap with tarsum. - tarsumLayerData, err := tarsum.NewTarSum(layerData, true, tarsum.Version0) + tarsumLayerData, err := tarsum.NewTarSum(layerData, true, tarsum.VersionDev) if err != nil { return err }