From bffd523279a28abff65b61ea9355d84b3499286e Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 29 Aug 2014 08:32:09 -0700 Subject: [PATCH 1/6] plumbing v2 `docker push` --- graph/pull.go | 14 ++++++++++++++ graph/push.go | 1 - 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/graph/pull.go b/graph/pull.go index 06a22c0f40bfb..a5adb0c1694e2 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -66,6 +66,20 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { localName = remoteName } + // XXX + // 1) get the image Manifest + // 1.a) validate the signature and payload of the manifest + // 2) then pull the blobs from the manifest + // 2.a) for docker-v1 compat, the map[sum]jsonBytes will need to be unpacked to determine the + // image ID and path to store the tar and json in + if len(tag) == 0 { + tag = DEFAULTTAG + } + if endpoint.Version == registry.APIVersion2 { + log.Debugf("SUCH WHOOP WHOOP") + return engine.StatusOK // return from this pull, so we don't do a v1 pull + } + if err = s.pullRepository(r, job.Stdout, localName, remoteName, tag, sf, job.GetenvBool("parallel")); err != nil { return job.Error(err) } diff --git a/graph/push.go b/graph/push.go index 936c1389076fc..3b67d93664685 100644 --- a/graph/push.go +++ b/graph/push.go @@ -239,7 +239,6 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { // 1.c) if anything else, err // 2) PUT the created/signed manifest if endpoint.Version == registry.APIVersion2 { - log.Debugf("SUCH WHOOP WHOOP") // for each layer, check if it exists ... // XXX wait this requires having the TarSum of the layer.tar first // skip this step for now. Just push the layer every time for this naive implementation From f19b73ce30917c726f7b58ae6c5d82ca7a8408bc Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 29 Aug 2014 11:07:23 -0700 Subject: [PATCH 2/6] Update pull to use manifest data from registry --- graph/pull.go | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index fb8a8d8c469f4..d12c7f54b26a7 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -84,26 +84,15 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - manifest := map[string]interface{}{} + var manifest registry.ManifestData err = json.Unmarshal(manifestBytes, &manifest) if err != nil { return job.Error(err) } - log.Debugf("%#v", manifest["history"]) - h, ok := manifest["history"].(map[string]interface{}) - if !ok { - return job.Error(fmt.Errorf("manifest 'history' is not a map[string]string")) - } - log.Debugf("%#v", manifest["tarsum"]) - sums, ok := manifest["tarsum"].([]interface{}) - if !ok { - return job.Error(fmt.Errorf("manifest 'tarsum' is not a []string")) - } - for _, sumInterface := range sums { - sumStr := sumInterface.(string) - jsonBytes := h[sumStr] - // - _ = jsonBytes.(string) + for _, sumStr := range manifest.BlobSums { + //jsonBytes := manifest.History[sumStr] + //// + //_ = jsonBytes.(string) chunks := strings.SplitN(sumStr, ":", 2) if len(chunks) < 2 { return job.Error(fmt.Errorf("expected 2 parts in the sumStr, got %#v", chunks)) @@ -119,7 +108,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { fmt.Println(tmpFile) } - log.Debugf("%#v", manifest["history"]) + log.Debugf("%#v", manifest.History) return engine.StatusOK // return from this pull, so we don't do a v1 pull } From 171de01734d78c6e6a2f4ad1b17fbbc1c04827ce Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Fri, 29 Aug 2014 11:08:42 -0700 Subject: [PATCH 3/6] vendored libtrust --- vendor/src/github.com/docker/libtrust | 1 + 1 file changed, 1 insertion(+) create mode 160000 vendor/src/github.com/docker/libtrust diff --git a/vendor/src/github.com/docker/libtrust b/vendor/src/github.com/docker/libtrust new file mode 160000 index 0000000000000..540d8fbb3dc14 --- /dev/null +++ b/vendor/src/github.com/docker/libtrust @@ -0,0 +1 @@ +Subproject commit 540d8fbb3dc145c58b0f3d44c1c41f087c6eacf5 From f0500a5ac5e2dcf73d60599e9214ca668dc8fcf1 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Fri, 29 Aug 2014 17:34:17 -0700 Subject: [PATCH 4/6] Working impl. of push/pull - no sig. verification yet --- graph/manifest.go | 7 +++++-- graph/pull.go | 40 +++++++++++++++++++++++++++++++--------- registry/session_prov.go | 4 ++-- registry/types.go | 10 +++++----- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/graph/manifest.go b/graph/manifest.go index 401aad8bdc175..7acf719346ea6 100644 --- a/graph/manifest.go +++ b/graph/manifest.go @@ -46,10 +46,13 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { layersSeen := make(map[string]bool) layer, err := s.graph.Get(layerId) + if err != nil { + return job.Error(err) + } manifest.Architecture = layer.Architecture var metadata runconfig.Config metadata = *layer.Config - history := make(map[string]string) + history := make([]string, 0, cap(tarsums)) for ; layer != nil; layer, err = layer.GetParent() { if err != nil { @@ -83,7 +86,7 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { if err != nil { return job.Error(fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err)) } - history[tarId] = string(jsonData) + history = append(history, string(jsonData)) } manifest.BlobSums = tarsums diff --git a/graph/pull.go b/graph/pull.go index d12c7f54b26a7..316eae05c8cc8 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -89,26 +89,47 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - for _, sumStr := range manifest.BlobSums { - //jsonBytes := manifest.History[sumStr] - //// - //_ = jsonBytes.(string) + + if len(manifest.BlobSums) != len(manifest.History) { + return job.Errorf("length of history not equal to number of layers") + } + + for i := len(manifest.BlobSums) - 1; i >= 0; i-- { + sumStr := manifest.BlobSums[i] + imgJSON := []byte(manifest.History[i]) + + img, err := image.NewImgJSON(imgJSON) + if err != nil { + return job.Error(fmt.Errorf("failed to parse json: %s", err)) + } + chunks := strings.SplitN(sumStr, ":", 2) if len(chunks) < 2 { return job.Error(fmt.Errorf("expected 2 parts in the sumStr, got %#v", chunks)) } + sumType, checksum := chunks[0], chunks[1] + + log.Infof("pulling blob %q to V1 img %s", sumStr, img.ID) tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob") if err != nil { - job.Error(err) + return job.Error(err) } - if err = r.GetV2ImageBlob(remoteName, chunks[0], chunks[1], tmpFile, nil); err != nil { - job.Error(err) + if err = r.GetV2ImageBlob(remoteName, sumType, checksum, tmpFile, nil); err != nil { + return job.Error(err) } fmt.Println(tmpFile) - } + tmpFile.Seek(0, 0) - log.Debugf("%#v", manifest.History) + err = s.graph.Register([]byte(imgJSON), tmpFile, img) + if err != nil { + return job.Error(err) + } + + if err = s.Set(localName, tag, img.ID, true); err != nil { + return job.Error(err) + } + } return engine.StatusOK // return from this pull, so we don't do a v1 pull } @@ -296,6 +317,7 @@ func (s *TagStore) pullImage(r *registry.Session, out io.Writer, imgID, endpoint continue } img, err = image.NewImgJSON(imgJSON) + // _RETURN HERE after getting image fom json if err != nil && j == retries { out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil)) return fmt.Errorf("Failed to parse json: %s", err) diff --git a/registry/session_prov.go b/registry/session_prov.go index 0eec9cab3698b..ef81d0562365b 100644 --- a/registry/session_prov.go +++ b/registry/session_prov.go @@ -198,11 +198,11 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri return err } defer res.Body.Close() - if res.StatusCode != 201 { + if res.StatusCode != 200 { if res.StatusCode == 401 { return errLoginRequired } - return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to pull %s blob", res.StatusCode, imageName), res) } _, err = io.Copy(blobWrtr, res.Body) diff --git a/registry/types.go b/registry/types.go index f71833f76be12..db246ab8edc5a 100644 --- a/registry/types.go +++ b/registry/types.go @@ -33,11 +33,11 @@ type RegistryInfo struct { } type ManifestData struct { - Name string `json:"name"` - Tag string `json:"tag"` - Architecture string `json:"architecture"` - BlobSums []string `json:"blobSums"` - History map[string]string `json:"history"` + Name string `json:"name"` + Tag string `json:"tag"` + Architecture string `json:"architecture"` + BlobSums []string `json:"blobSums"` + History []string `json:"history"` } type APIVersion int From 95532571820ae399cd1cb237f03cb9561364cbab Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 3 Sep 2014 14:18:26 -0700 Subject: [PATCH 5/6] Add signature verification --- graph/pull.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index 316eae05c8cc8..788d8f9c3075d 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -15,6 +15,7 @@ import ( "github.com/docker/docker/pkg/log" "github.com/docker/docker/registry" "github.com/docker/docker/utils" + "github.com/docker/libtrust" ) func (s *TagStore) CmdPull(job *engine.Job) engine.Status { @@ -85,11 +86,20 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { return job.Error(err) } var manifest registry.ManifestData - err = json.Unmarshal(manifestBytes, &manifest) + + sig, err := libtrust.ParsePrettySignature(manifestBytes, "buildSignatures") if err != nil { - return job.Error(err) + return job.Errorf("error parsing signature: %s", err) + } + _, err = sig.Verify() + if err != nil { + return job.Errorf("error verifying signature: %s", err) } + err = json.Unmarshal(manifestBytes, &manifest) + if err != nil { + return job.Errorf("error unmarshalling manifest: %s", err) + } if len(manifest.BlobSums) != len(manifest.History) { return job.Errorf("length of history not equal to number of layers") } From a9007c922e9d9554450ae096d0e2fb01c1ae5da8 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 8 Sep 2014 10:54:40 -0700 Subject: [PATCH 6/6] Move verification logic to function, use single "signatures" key --- api/client/commands.go | 2 +- graph/pull.go | 37 ++++++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/api/client/commands.go b/api/client/commands.go index 6eb440259832c..fbc90ff7149ab 100644 --- a/api/client/commands.go +++ b/api/client/commands.go @@ -1169,7 +1169,7 @@ func (cli *DockerCli) CmdPush(args ...string) error { return err } - signedBody, err := js.PrettySignature("buildSignatures") + signedBody, err := js.PrettySignature("signatures") if err != nil { return err } diff --git a/graph/pull.go b/graph/pull.go index 788d8f9c3075d..121052ea18368 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -18,6 +18,30 @@ import ( "github.com/docker/libtrust" ) +func (s *TagStore) verifyManifest(manifestBytes []byte) (*registry.ManifestData, error) { + sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures") + if err != nil { + return nil, fmt.Errorf("error parsing payload: %s", err) + } + _, err = sig.Verify() + if err != nil { + return nil, fmt.Errorf("error verifying payload: %s", err) + } + + payload, err := sig.Payload() + if err != nil { + return nil, fmt.Errorf("error retrieving payload: %s", err) + } + + var manifest registry.ManifestData + err = json.Unmarshal(payload, &manifest) + if err != nil { + return nil, fmt.Errorf("error unmarshalling manifest: %s", err) + } + + return &manifest, nil +} + func (s *TagStore) CmdPull(job *engine.Job) engine.Status { if n := len(job.Args); n != 1 && n != 2 { return job.Errorf("Usage: %s IMAGE [TAG]", job.Name) @@ -85,21 +109,12 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - var manifest registry.ManifestData - sig, err := libtrust.ParsePrettySignature(manifestBytes, "buildSignatures") + manifest, err := s.verifyManifest(manifestBytes) if err != nil { - return job.Errorf("error parsing signature: %s", err) - } - _, err = sig.Verify() - if err != nil { - return job.Errorf("error verifying signature: %s", err) + return job.Errorf("error verifying manifest: %s", err) } - err = json.Unmarshal(manifestBytes, &manifest) - if err != nil { - return job.Errorf("error unmarshalling manifest: %s", err) - } if len(manifest.BlobSums) != len(manifest.History) { return job.Errorf("length of history not equal to number of layers") }