Calculate checksum when installing image layers

daemon/graphdriver/aufs/aufs.go

@@ -300,11 +300,14 @@ func (a *Driver) Diff(id, parent string) (archive.Archive, error) {
 	// AUFS doesn't need the parent layer to produce a diff.
 	return archive.TarWithOptions(path.Join(a.rootPath(), "diff", id), &archive.TarOptions{
 		Compression: archive.Uncompressed,
+		Excludes:    []string{".wh..wh.*"},
 	})
 }
 
 func (a *Driver) applyDiff(id string, diff archive.ArchiveReader) error {
-	return archive.Untar(diff, path.Join(a.rootPath(), "diff", id), nil)
+	return archive.Untar(diff, path.Join(a.rootPath(), "diff", id), &archive.TarOptions{
+		Excludes: []string{".wh..wh."},
+	})
 }
 
 // DiffSize calculates the changes between the specified id

graph/graph.go

@@ -132,14 +132,14 @@ func (graph *Graph) Create(layerData archive.ArchiveReader, containerID, contain
 		img.ContainerConfig = *containerConfig
 	}
 
-	if err := graph.Register(img, nil, layerData); err != nil {
+	if err := graph.Register(img, layerData); err != nil {
 		return nil, err
 	}
 	return img, nil
 }
 
 // Register imports a pre-existing image into the graph.
-func (graph *Graph) Register(img *image.Image, jsonData []byte, layerData archive.ArchiveReader) (err error) {
+func (graph *Graph) Register(img *image.Image, layerData archive.ArchiveReader) (err error) {
 	defer func() {
 		// If any error occurs, remove the new dir from the driver.
 		// Don't check for errors since the dir might not have been created.
@@ -181,7 +181,7 @@ func (graph *Graph) Register(img *image.Image, jsonData []byte, layerData archiv
 	}
 	// Apply the diff/layer
 	img.SetGraph(graph)
-	if err := image.StoreImage(img, jsonData, layerData, tmp); err != nil {
+	if err := image.StoreImage(img, layerData, tmp); err != nil {
 		return err
 	}
 	// Commit

graph/load.go

@@ -118,7 +118,7 @@ func (s *TagStore) recursiveLoad(eng *engine.Engine, address, tmpImageDir string
 				}
 			}
 		}
-		if err := s.graph.Register(img, imageJson, layer); err != nil {
+		if err := s.graph.Register(img, layer); err != nil {
 			return err
 		}
 	}

graph/pull.go

@@ -392,8 +392,7 @@ func (s *TagStore) pullImage(r *registry.Session, out io.Writer, imgID, endpoint
 				layers_downloaded = true
 				defer layer.Close()
 
-				err = s.graph.Register(img, imgJSON,
-					utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading"))
+				err = s.graph.Register(img, utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading"))
 				if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
 					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
 					continue
@@ -577,7 +576,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri
 			defer d.tmpFile.Close()
 			d.tmpFile.Seek(0, 0)
 			if d.tmpFile != nil {
-				err = s.graph.Register(d.img, d.imgJSON,
+				err = s.graph.Register(d.img,
 					utils.ProgressReader(d.tmpFile, int(d.length), out, sf, false, utils.TruncateID(d.img.ID), "Extracting"))
 				if err != nil {
 					return false, err

graph/service.go

@@ -6,12 +6,10 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/docker/engine"
-	"github.com/docker/docker/image"
 )
 
 func (s *TagStore) Install(eng *engine.Engine) error {
 	for name, handler := range map[string]engine.Handler{
-		"image_set":      s.CmdSet,
 		"image_tag":      s.CmdTag,
 		"tag":            s.CmdTagLegacy, // FIXME merge with "image_tag"
 		"image_get":      s.CmdGet,
@@ -33,53 +31,6 @@ func (s *TagStore) Install(eng *engine.Engine) error {
 	return nil
 }
 
-// CmdSet stores a new image in the graph.
-// Images are stored in the graph using 4 elements:
-//	- A user-defined ID
-//	- A collection of metadata describing the image
-//	- A directory tree stored as a tar archive (also called the "layer")
-//	- A reference to a "parent" ID on top of which the layer should be applied
-//
-// NOTE: even though the parent ID is only useful in relation to the layer and how
-// to apply it (ie you could represent the full directory tree as 'parent_layer + layer',
-// it is treated as a top-level property of the image. This is an artifact of early
-// design and should probably be cleaned up in the future to simplify the design.
-//
-// Syntax: image_set ID
-// Input:
-//	- Layer content must be streamed in tar format on stdin. An empty input is
-//	valid and represents a nil layer.
-//
-//	- Image metadata must be passed in the command environment.
-//		'json': a json-encoded object with all image metadata.
-//			It will be stored as-is, without any encoding/decoding artifacts.
-//			That is a requirement of the current registry client implementation,
-//			because a re-encoded json might invalidate the image checksum at
-//			the next upload, even with functionaly identical content.
-func (s *TagStore) CmdSet(job *engine.Job) engine.Status {
-	if len(job.Args) != 1 {
-		return job.Errorf("usage: %s NAME", job.Name)
-	}
-	var (
-		imgJSON = []byte(job.Getenv("json"))
-		layer   = job.Stdin
-	)
-	if len(imgJSON) == 0 {
-		return job.Errorf("mandatory key 'json' is not set")
-	}
-	// We have to pass an *image.Image object, even though it will be completely
-	// ignored in favor of the redundant json data.
-	// FIXME: the current prototype of Graph.Register is stupid and redundant.
-	img, err := image.NewImgJSON(imgJSON)
-	if err != nil {
-		return job.Error(err)
-	}
-	if err := s.graph.Register(img, imgJSON, layer); err != nil {
-		return job.Error(err)
-	}
-	return engine.StatusOK
-}
-
 // CmdGet returns information about an image.
 // If the image doesn't exist, an empty object is returned, to allow
 // checking for an image's existence.
@@ -150,6 +101,7 @@ func (s *TagStore) CmdLookup(job *engine.Job) engine.Status {
 		out.Set("Os", image.OS)
 		out.SetInt64("Size", image.Size)
 		out.SetInt64("VirtualSize", image.GetParentsSize(0)+image.Size)
+		out.Set("Checksum", image.Checksum)
 		if _, err = out.WriteTo(job.Stdout); err != nil {
 			return job.Error(err)
 		}

graph/tags_unit_test.go

@@ -62,7 +62,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore {
 		t.Fatal(err)
 	}
 	img := &image.Image{ID: testImageID}
-	if err := graph.Register(img, nil, archive); err != nil {
+	if err := graph.Register(img, archive); err != nil {
 		t.Fatal(err)
 	}
 	if err := store.Set(testImageName, "", testImageID, false); err != nil {

image/image.go

@@ -11,6 +11,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/tarsum"
 	"github.com/docker/docker/runconfig"
 	"github.com/docker/docker/utils"
 )
@@ -32,6 +33,7 @@ type Image struct {
 	Config          *runconfig.Config `json:"config,omitempty"`
 	Architecture    string            `json:"architecture,omitempty"`
 	OS              string            `json:"os,omitempty"`
+	Checksum        string            `json:"checksum"`
 	Size            int64
 
 	graph Graph
@@ -70,7 +72,7 @@ func LoadImage(root string) (*Image, error) {
 	return img, nil
 }
 
-func StoreImage(img *Image, jsonData []byte, layerData archive.ArchiveReader, root string) error {
+func StoreImage(img *Image, layerData archive.ArchiveReader, root string) error {
 	// Store the layer
 	var (
 		size   int64
@@ -80,29 +82,43 @@ func StoreImage(img *Image, jsonData []byte, layerData archive.ArchiveReader, ro
 
 	// If layerData is not nil, unpack it into the new layer
 	if layerData != nil {
-		if size, err = driver.ApplyDiff(img.ID, img.Parent, layerData); err != nil {
+		// Autodetect compression of the layer data.
+		layerData, err := archive.DecompressStream(layerData)
+		if err != nil {
+			return err
+		}
+		defer layerData.Close()
+
+		// Wrap with tarsum.
+		tarsumLayerData, err := tarsum.NewTarSum(layerData, true, tarsum.VersionDev)
+		if err != nil {
 			return err
 		}
+
+		// Extract the archive to the file system driver.
+		if size, err = driver.ApplyDiff(img.ID, img.Parent, tarsumLayerData); err != nil {
+			return err
+		}
+
+		// Get the resulting tarsum.
+		img.Checksum = tarsumLayerData.Sum(nil)
 	}
 
 	img.Size = size
 	if err := img.SaveSize(root); err != nil {
 		return err
 	}
 
-	// If raw json is provided, then use it
-	if jsonData != nil {
-		if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil {
-			return err
-		}
-	} else {
-		if jsonData, err = json.Marshal(img); err != nil {
-			return err
-		}
-		if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil {
-			return err
-		}
+	var jsonData []byte
+
+	if jsonData, err = json.Marshal(img); err != nil {
+		return err
 	}
+
+	if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil {
+		return err
+	}
+
 	return nil
 }
 

integration/graph_test.go

@@ -74,7 +74,7 @@ func TestInterruptedRegister(t *testing.T) {
 		Created: time.Now(),
 	}
 	w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling)
-	graph.Register(image, nil, badArchive)
+	graph.Register(image, badArchive)
 	if _, err := graph.Get(image.ID); err == nil {
 		t.Fatal("Image should not exist after Register is interrupted")
 	}
@@ -83,7 +83,7 @@ func TestInterruptedRegister(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := graph.Register(image, nil, goodArchive); err != nil {
+	if err := graph.Register(image, goodArchive); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -133,7 +133,7 @@ func TestRegister(t *testing.T) {
 		Comment: "testing",
 		Created: time.Now(),
 	}
-	err = graph.Register(image, nil, archive)
+	err = graph.Register(image, archive)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -228,7 +228,7 @@ func TestDelete(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Test delete twice (pull -> rm -> pull -> rm)
-	if err := graph.Register(img1, nil, archive); err != nil {
+	if err := graph.Register(img1, archive); err != nil {
 		t.Fatal(err)
 	}
 	if err := graph.Delete(img1.ID); err != nil {
@@ -262,9 +262,9 @@ func TestByParent(t *testing.T) {
 		Created: time.Now(),
 		Parent:  parentImage.ID,
 	}
-	_ = graph.Register(parentImage, nil, archive1)
-	_ = graph.Register(childImage1, nil, archive2)
-	_ = graph.Register(childImage2, nil, archive3)
+	_ = graph.Register(parentImage, archive1)
+	_ = graph.Register(childImage1, archive2)
+	_ = graph.Register(childImage2, archive3)
 
 	byParent, err := graph.ByParent()
 	if err != nil {