From c003172afe169b7275838ce50d40cc32691fe6bb Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Thu, 28 Aug 2014 15:43:52 -0700 Subject: [PATCH] Added registry v2 server impl --- registry/v2/registry/server.go | 21 ++++ registry/v2/routes/router.go | 36 +++++++ registry/v2/server/blobs.go | 151 +++++++++++++++++++++++++++++ registry/v2/server/manifests.go | 122 +++++++++++++++++++++++ registry/v2/server/mount_blob.go | 58 +++++++++++ registry/v2/server/set_handlers.go | 45 +++++++++ registry/v2/server/storage.go | 12 +++ registry/v2/server/sum_readers.go | 113 +++++++++++++++++++++ registry/v2/server/tags.go | 56 +++++++++++ 9 files changed, 614 insertions(+) create mode 100644 registry/v2/registry/server.go create mode 100644 registry/v2/routes/router.go create mode 100644 registry/v2/server/blobs.go create mode 100644 registry/v2/server/manifests.go create mode 100644 registry/v2/server/mount_blob.go create mode 100644 registry/v2/server/set_handlers.go create mode 100644 registry/v2/server/storage.go create mode 100644 registry/v2/server/sum_readers.go create mode 100644 registry/v2/server/tags.go diff --git a/registry/v2/registry/server.go b/registry/v2/registry/server.go new file mode 100644 index 0000000000000..d013e98d20629 --- /dev/null +++ b/registry/v2/registry/server.go @@ -0,0 +1,21 @@ +package main + +import ( + "log" + "net/http" + "time" + + registryServer "github.com/docker/docker/registry/v2/server" +) + +func main() { + server := &http.Server{ + Addr: ":8080", + Handler: registryServer.NewRegistryHandler(), + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + } + + log.Fatal(server.ListenAndServe()) +} diff --git a/registry/v2/routes/router.go b/registry/v2/routes/router.go new file mode 100644 index 0000000000000..6b182140773aa --- /dev/null +++ b/registry/v2/routes/router.go @@ -0,0 +1,36 @@ +package routes + +import ( + "github.com/gorilla/mux" +) + +const ( + ManifestsRouteName = "manifests" + TagsRouteName = "tags" + DownloadBlobRouteName = "downloadBlob" + UploadBlobRouteName = "uploadBlob" + MountBlobRouteName = "mountBlob" +) + +func NewRegistryRouter() *mux.Router { + router := mux.NewRouter() + + v2Route := router.PathPrefix("/v2/").Subrouter() + + // Image Manifests + v2Route.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name(ManifestsRouteName) + + // List Image Tags + v2Route.Path("/tags/{imagename:[a-z0-9-._/]+}").Name(TagsRouteName) + + // Download a blob + v2Route.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name(DownloadBlobRouteName) + + // Upload a blob + v2Route.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}").Name(UploadBlobRouteName) + + // Mounting a blob in an image + v2Route.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9_+-]+}/{sum:[a-fA-F0-9]{4,}}").Name(MountBlobRouteName) + + return router +} diff --git a/registry/v2/server/blobs.go b/registry/v2/server/blobs.go new file mode 100644 index 0000000000000..4fc8ece07d541 --- /dev/null +++ b/registry/v2/server/blobs.go @@ -0,0 +1,151 @@ +package server + +import ( + "encoding/json" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path" + "strings" + + "github.com/gorilla/mux" +) + +func getBlob(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + log.Printf("Get Blob: %#v\n", vars) + + _, ok := vars["imagename"] + if !ok { + w.WriteHeader(404) + return + } + + sumType, ok := vars["sumtype"] + if !ok { + w.WriteHeader(404) + return + } + + sum, ok := vars["sum"] + if !ok { + w.WriteHeader(404) + return + } + + prefix1, prefix2 := sum[:2], sum[2:4] + + blobPath := path.Join(blobsDirectory, sumType, prefix1, prefix2, sum) + blobFile, err := os.Open(blobPath) + if err != nil { + errStatus := 500 + if os.IsNotExist(err) { + errStatus = 404 + } + log.Printf("unable to open blob file %q: %s\n", blobPath, err) + w.WriteHeader(errStatus) + return + } + + bytesCopied, err := io.Copy(w, blobFile) + if err != nil { + log.Printf("unable to copy blob file %q: %s\n", blobPath, err) + w.WriteHeader(500) + } else { + log.Printf("copied %d bytes from blob file %q\n", bytesCopied, blobPath) + } +} + +func putBlob(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + log.Printf("Put Blob: %#v\n", vars) + + _, ok := vars["imagename"] + if !ok { + w.WriteHeader(404) + return + } + + sumType, ok := vars["sumtype"] + if !ok { + w.WriteHeader(404) + return + } + + blobDir := path.Join(blobsDirectory, sumType) + err := os.MkdirAll(blobDir, os.FileMode(0755)) + if err != nil { + log.Printf("unable to create blob directory %q: %s\n", blobDir, err) + w.WriteHeader(500) + return + } + + tempBlobFile, err := ioutil.TempFile(blobDir, "temp") + if err != nil { + log.Printf("unable to open temporary blob file %q: %s\n", tempBlobFile.Name(), err) + w.WriteHeader(500) + return + } + + sumReader, err := NewSumReader(sumType, io.TeeReader(r.Body, tempBlobFile)) + if err != nil { + log.Printf("unable to create %q sum reader: %s\n", sumType, err) + tempBlobFile.Close() + os.Remove(tempBlobFile.Name()) + if err == ErrSumTypeNotSupported { + // sumType is not Supported + w.WriteHeader(501) + } else { + // content type must not be what the sumReader expects. + w.WriteHeader(400) + } + return + } + + bytesCopied, err := io.Copy(ioutil.Discard, sumReader) + tempBlobFile.Close() + sumReader.Close() + if err != nil { + log.Printf("unable to copy request body to temp blob file %q: %s\n", tempBlobFile.Name(), err) + // Delete temp file. + os.Remove(tempBlobFile.Name()) + w.WriteHeader(500) + return + } + + log.Printf("copied %d bytes from request body to temp blob file %q\n", bytesCopied, tempBlobFile.Name()) + + type sumReturn struct { + Checksum string `json:"checksum"` + } + + sumInfo := sumReturn{ + Checksum: strings.ToLower(sumReader.Sum(nil)), + } + + // Split on the sumType delimiter to get the sum value. + sum := strings.SplitN(sumInfo.Checksum, ":", 2)[1] + + prefix1, prefix2 := sum[:2], sum[2:4] + + blobDir = path.Join(blobsDirectory, sumType, prefix1, prefix2) + err = os.MkdirAll(blobDir, os.FileMode(0755)) + if err != nil { + log.Printf("unable to create blob directory %q: %s\n", blobDir, err) + os.Remove(tempBlobFile.Name()) + w.WriteHeader(500) + return + } + + // Rename temp file. + blobPath := path.Join(blobDir, sum) + os.Rename(tempBlobFile.Name(), blobPath) + // Set 201 Header. + w.WriteHeader(201) + + // Write JSON body. + encoder := json.NewEncoder(w) + encoder.Encode(sumInfo) +} diff --git a/registry/v2/server/manifests.go b/registry/v2/server/manifests.go new file mode 100644 index 0000000000000..28dc706abc0dc --- /dev/null +++ b/registry/v2/server/manifests.go @@ -0,0 +1,122 @@ +package server + +import ( + "io" + "log" + "net/http" + "os" + "path" + + "github.com/gorilla/mux" +) + +func getManifest(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + log.Printf("Get Manifest: %#v\n", vars) + + imageName, ok := vars["imagename"] + if !ok { + w.WriteHeader(404) + return + } + + tagName, ok := vars["tagname"] + if !ok { + w.WriteHeader(404) + return + } + + manifestPath := path.Join(imagesDirectory, imageName, tagName) + manifestFile, err := os.Open(manifestPath) + if err != nil { + errStatus := 500 + if os.IsNotExist(err) { + errStatus = 404 + } + log.Printf("unable to open manifest file %q: %s\n", manifestPath, err) + w.WriteHeader(errStatus) + return + } + + bytesCopied, err := io.Copy(w, manifestFile) + if err != nil { + log.Printf("unable to copy manifest file %q: %s\n", manifestPath, err) + w.WriteHeader(500) + } else { + log.Printf("copied %d bytes from manifest file %q\n", bytesCopied, manifestPath) + } +} + +func putManifest(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + log.Printf("Put Manifest: %#v\n", vars) + + imageName, ok := vars["imagename"] + if !ok { + w.WriteHeader(404) + return + } + + tagName, ok := vars["tagname"] + if !ok { + w.WriteHeader(404) + return + } + + manifestDir := path.Join(imagesDirectory, imageName) + err := os.MkdirAll(manifestDir, os.FileMode(0755)) + if err != nil { + log.Printf("unable to create manifest directory %q: %s\n", manifestDir, err) + w.WriteHeader(500) + return + } + + manifestPath := path.Join(manifestDir, tagName) + manifestFile, err := os.OpenFile(manifestPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(0644)) + if err != nil { + log.Printf("unable to open manifest file %q: %s\n", manifestPath, err) + w.WriteHeader(500) + return + } + + bytesCopied, err := io.Copy(manifestFile, r.Body) + if err != nil { + log.Printf("unable to copy request body to manifest file %q: %s\n", manifestPath, err) + w.WriteHeader(500) + } else { + log.Printf("copied %d bytes from request body to manifest file %q\n", bytesCopied, manifestPath) + } + + w.WriteHeader(201) +} + +func deleteManifest(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + log.Printf("Delete Manifest: %#v\n", vars) + + imageName, ok := vars["imagename"] + if !ok { + w.WriteHeader(404) + return + } + + tagName, ok := vars["tagname"] + if !ok { + w.WriteHeader(404) + return + } + + manifestPath := path.Join(imagesDirectory, imageName, tagName) + err := os.Remove(manifestPath) + if err != nil { + errStatus := 500 + if os.IsNotExist(err) { + errStatus = 404 + } + log.Printf("unable to remove manifest file %q: %s\n", manifestPath, err) + w.WriteHeader(errStatus) + return + } + + w.WriteHeader(204) +} diff --git a/registry/v2/server/mount_blob.go b/registry/v2/server/mount_blob.go new file mode 100644 index 0000000000000..49cd7841119a2 --- /dev/null +++ b/registry/v2/server/mount_blob.go @@ -0,0 +1,58 @@ +package server + +import ( + "log" + "net/http" + "os" + "path" + + "github.com/gorilla/mux" +) + +func mountBlob(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + log.Printf("Mount Blob: %#v\n", vars) + + _, ok := vars["imagename"] + if !ok { + w.WriteHeader(404) + return + } + + sumType, ok := vars["sumtype"] + if !ok { + w.WriteHeader(404) + return + } + + sum, ok := vars["sum"] + if !ok { + w.WriteHeader(404) + return + } + + prefix1, prefix2 := sum[:2], sum[2:4] + + blobPath := path.Join(blobsDirectory, sumType, prefix1, prefix2, sum) + fileInfo, err := os.Lstat(blobPath) + if err != nil { + errStatus := 500 + if os.IsNotExist(err) { + // The blob does not exist. Indicate to the client that they should upload it. + errStatus = 300 + } + log.Printf("unable to open blob file %q: %s\n", blobPath, err) + w.WriteHeader(errStatus) + return + } + + if !fileInfo.Mode().IsRegular() { + log.Printf("unable to associate blob file %q: not a regular file", blobPath) + w.WriteHeader(500) + return + } + + // The blob exists and is a regular file! On this naive server, that's OK. + // We don't really have access control lists to worry about, everything is public. + // TODO: return some content. +} diff --git a/registry/v2/server/set_handlers.go b/registry/v2/server/set_handlers.go new file mode 100644 index 0000000000000..30a4c2770b23a --- /dev/null +++ b/registry/v2/server/set_handlers.go @@ -0,0 +1,45 @@ +package server + +import ( + "net/http" + + "github.com/docker/docker/registry/v2/routes" + "github.com/gorilla/mux" +) + +func setRegistryRouteHandlers(registryRouter *mux.Router) { + routeHandlers := map[string]map[string]http.Handler{ + routes.ManifestsRouteName: { + "GET": http.HandlerFunc(getManifest), + "PUT": http.HandlerFunc(putManifest), + "DELETE": http.HandlerFunc(deleteManifest), + }, + routes.TagsRouteName: { + "GET": http.HandlerFunc(getTags), + }, + routes.DownloadBlobRouteName: { + "GET": http.HandlerFunc(getBlob), + }, + routes.UploadBlobRouteName: { + "PUT": http.HandlerFunc(putBlob), + }, + routes.MountBlobRouteName: { + "POST": http.HandlerFunc(mountBlob), + }, + } + + for routeName, handlerMapping := range routeHandlers { + route := registryRouter.Get(routeName) + + subRouter := route.Subrouter() + for methodName, handler := range handlerMapping { + subRouter.Methods(methodName).Handler(handler) + } + } +} + +func NewRegistryHandler() http.Handler { + router := routes.NewRegistryRouter() + setRegistryRouteHandlers(router) + return router +} diff --git a/registry/v2/server/storage.go b/registry/v2/server/storage.go new file mode 100644 index 0000000000000..ff538667d7eaa --- /dev/null +++ b/registry/v2/server/storage.go @@ -0,0 +1,12 @@ +package server + +import ( + "os" + "path" +) + +var ( + dataDirectory = os.ExpandEnv("$HOME/registry_data") + blobsDirectory = path.Join(dataDirectory, "blobs") + imagesDirectory = path.Join(dataDirectory, "images") +) diff --git a/registry/v2/server/sum_readers.go b/registry/v2/server/sum_readers.go new file mode 100644 index 0000000000000..203660e8d9e69 --- /dev/null +++ b/registry/v2/server/sum_readers.go @@ -0,0 +1,113 @@ +package server + +import ( + "compress/gzip" + "crypto" + "encoding/hex" + "errors" + "hash" + "io" + + "github.com/docker/docker/pkg/tarsum" +) + +// SumReader is able to read data from an internal io.Reader +// and produce a checksum of the data which has been read. +type SumReader interface { + Sum(extra []byte) string + Read(p []byte) (n int, err error) + Close() error +} + +var ( + supportedSumTypes map[string]func(io.Reader) (SumReader, error) + // ErrSumTypeNotSupported indicates that the sum type is not supported. + ErrSumTypeNotSupported = errors.New("sum type not supported") +) + +// NewSumReader returns an instance of a SumReader which +// implements the given checksum type. Returns a nil +// SumReader if the given sum type is not supported. +func NewSumReader(sumTypeName string, r io.Reader) (SumReader, error) { + contstructor, ok := supportedSumTypes[sumTypeName] + if !ok { + return nil, ErrSumTypeNotSupported + } + + return contstructor(r) +} + +// HashingSumReader implements SumWriter +// using a cryptographic hashing algorithm. +// The Read method is covered by the internal +// io.TeeReader +type HashingSumReader struct { + io.Reader + hash hash.Hash + label string +} + +// Sum returns the current cryptographic hash +// digest of the data which has been written. +// The format is "