-
Notifications
You must be signed in to change notification settings - Fork 2
/
push.go
341 lines (295 loc) · 10.3 KB
/
push.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
package graph
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"github.com/docker/docker/archive"
"github.com/docker/docker/engine"
"github.com/docker/docker/pkg/log"
"github.com/docker/docker/pkg/tarsum"
"github.com/docker/docker/registry"
"github.com/docker/docker/utils"
)
// Retrieve the all the images to be uploaded in the correct order
func (s *TagStore) getImageList(localRepo map[string]string, requestedTag string) ([]string, map[string][]string, error) {
var (
imageList []string
imagesSeen map[string]bool = make(map[string]bool)
tagsByImage map[string][]string = make(map[string][]string)
)
for tag, id := range localRepo {
if requestedTag != "" && requestedTag != tag {
continue
}
var imageListForThisTag []string
tagsByImage[id] = append(tagsByImage[id], tag)
for img, err := s.graph.Get(id); img != nil; img, err = img.GetParent() {
if err != nil {
return nil, nil, err
}
if imagesSeen[img.ID] {
// This image is already on the list, we can ignore it and all its parents
break
}
imagesSeen[img.ID] = true
imageListForThisTag = append(imageListForThisTag, img.ID)
}
// reverse the image list for this tag (so the "most"-parent image is first)
for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 {
imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i]
}
// append to main image list
imageList = append(imageList, imageListForThisTag...)
}
if len(imageList) == 0 {
return nil, nil, fmt.Errorf("No images found for the requested repository / tag")
}
log.Debugf("Image list: %v", imageList)
log.Debugf("Tags by image: %v", tagsByImage)
return imageList, tagsByImage, nil
}
func (s *TagStore) pushRepository(r *registry.Session, out io.Writer, localName, remoteName string, localRepo map[string]string, tag string, sf *utils.StreamFormatter) error {
out = utils.NewWriteFlusher(out)
log.Debugf("Local repo: %s", localRepo)
imgList, tagsByImage, err := s.getImageList(localRepo, tag)
if err != nil {
return err
}
out.Write(sf.FormatStatus("", "Sending image list"))
var (
repoData *registry.RepositoryData
imageIndex []*registry.ImgData
)
for _, imgId := range imgList {
if tags, exists := tagsByImage[imgId]; exists {
// If an image has tags you must add an entry in the image index
// for each tag
for _, tag := range tags {
imageIndex = append(imageIndex, ®istry.ImgData{
ID: imgId,
Tag: tag,
})
}
} else {
// If the image does not have a tag it still needs to be sent to the
// registry with an empty tag so that it is accociated with the repository
imageIndex = append(imageIndex, ®istry.ImgData{
ID: imgId,
Tag: "",
})
}
}
log.Debugf("Preparing to push %s with the following images and tags", localRepo)
for _, data := range imageIndex {
log.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag)
}
// Register all the images in a repository with the registry
// If an image is not in this list it will not be associated with the repository
repoData, err = r.PushImageJSONIndex(remoteName, imageIndex, false, nil)
if err != nil {
return err
}
nTag := 1
if tag == "" {
nTag = len(localRepo)
}
for _, ep := range repoData.Endpoints {
out.Write(sf.FormatStatus("", "Pushing repository %s (%d tags)", localName, nTag))
for _, imgId := range imgList {
if r.LookupRemoteImage(imgId, ep, repoData.Tokens) {
out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", utils.TruncateID(imgId)))
} else {
if _, err := s.pushImage(r, out, remoteName, imgId, ep, repoData.Tokens, sf); err != nil {
// FIXME: Continue on error?
return err
}
}
for _, tag := range tagsByImage[imgId] {
out.Write(sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", utils.TruncateID(imgId), ep+"repositories/"+remoteName+"/tags/"+tag))
if err := r.PushRegistryTag(remoteName, imgId, tag, ep, repoData.Tokens); err != nil {
return err
}
}
}
}
if _, err := r.PushImageJSONIndex(remoteName, imageIndex, true, repoData.Endpoints); err != nil {
return err
}
return nil
}
func (s *TagStore) pushImage(r *registry.Session, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) (checksum string, err error) {
out = utils.NewWriteFlusher(out)
jsonRaw, err := ioutil.ReadFile(path.Join(s.graph.Root, imgID, "json"))
if err != nil {
return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err)
}
out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pushing", nil))
imgData := ®istry.ImgData{
ID: imgID,
}
// Send the json
if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil {
if err == registry.ErrAlreadyExists {
out.Write(sf.FormatProgress(utils.TruncateID(imgData.ID), "Image already pushed, skipping", nil))
return "", nil
}
return "", err
}
layerData, err := s.graph.TempLayerArchive(imgID, archive.Uncompressed, sf, out)
if err != nil {
return "", fmt.Errorf("Failed to generate layer archive: %s", err)
}
defer os.RemoveAll(layerData.Name())
// Send the layer
log.Debugf("rendered layer for %s of [%d] size", imgData.ID, layerData.Size)
checksum, checksumPayload, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf, false, utils.TruncateID(imgData.ID), "Pushing"), ep, token, jsonRaw)
if err != nil {
return "", err
}
imgData.Checksum = checksum
imgData.ChecksumPayload = checksumPayload
// Send the checksum
if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil {
return "", err
}
out.Write(sf.FormatProgress(utils.TruncateID(imgData.ID), "Image successfully pushed", nil))
return imgData.Checksum, nil
}
// FIXME: Allow to interrupt current push when new push of same image is done.
func (s *TagStore) CmdPush(job *engine.Job) engine.Status {
if n := len(job.Args); n != 1 {
return job.Errorf("Usage: %s IMAGE", job.Name)
}
var (
localName = job.Args[0]
sf = utils.NewStreamFormatter(job.GetenvBool("json"))
authConfig = ®istry.AuthConfig{}
metaHeaders map[string][]string
)
tag := job.Getenv("tag")
job.GetenvJson("authConfig", authConfig)
job.GetenvJson("metaHeaders", &metaHeaders)
if _, err := s.poolAdd("push", localName); err != nil {
return job.Error(err)
}
defer s.poolRemove("push", localName)
// Resolve the Repository name from fqn to endpoint + name
hostname, remoteName, err := registry.ResolveRepositoryName(localName)
if err != nil {
return job.Error(err)
}
endpoint, err := registry.NewEndpoint(hostname)
if err != nil {
return job.Error(err)
}
img, err := s.graph.Get(localName)
r, err2 := registry.NewSession(authConfig, registry.HTTPRequestFactory(metaHeaders), endpoint, false)
if err2 != nil {
return job.Error(err2)
}
if len(tag) == 0 {
tag = DEFAULTTAG
}
// XXX TESTING
// 1) Check if TarSum of each layer exists /v2/
// 1.a) if 200, continue
// 1.b) if 300, then push the
// 1.c) if anything else, err
// 2) PUT the created/signed manifest
if endpoint.Version == registry.APIVersion2 {
log.Debugf("SUCH WHOOP WHOOP")
// for each layer, check if it exists ...
// XXX wait this requires having the TarSum of the layer.tar first
// skip this step for now. Just push the layer every time for this naive implementation
//shouldPush, err := r.PostV2ImageMountBlob(imageName, sumType, sum string, token []string)
var manifestData = make(map[string][]byte)
// XXX unfortunately this goes from child to parent,
// but the list of blobs in the manifest is expected to go parent to child
localRepo := s.Repositories[localName]
imgList, tagsByImage, err := s.getImageList(localRepo, tag)
_ = tagsByImage // not yet ...
for _, imgID := range imgList {
img, err = s.graph.Get(imgID)
if err != nil {
return job.Error(err)
}
log.Debugf("SUCH IMAGE %#v", img)
arch, err := img.TarLayer()
if err != nil {
return job.Error(err)
}
tsRdr := &tarsum.TarSum{Reader: arch, DisableCompression: false}
log.Debugf("SUCH PUSH ID %s", img.ID)
serverChecksum, err := r.PutV2ImageBlob(remoteName, "tarsum+sha256", tsRdr, nil)
if err != nil {
return job.Error(err)
}
localChecksum := tsRdr.Sum(nil)
if serverChecksum != localChecksum {
return job.Error(fmt.Errorf("%q: failed checksum comparison. serverChecksum: %q, localChecksum: %q", remoteName, serverChecksum, localChecksum))
} else {
log.Debugf("imgID: %q, serverChecksum: %q, localChecksum: %q", img.ID, serverChecksum, localChecksum)
}
// So dumb. This should be a call to the image.Image RawJson()
manifestData[localChecksum], err = ioutil.ReadFile(path.Join(s.graph.Root, imgID, "json"))
if err != nil {
return job.Error(fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err))
}
}
// Next, produce the merged/flattened "image json"
// ...
// Next, produce the manifest
blobSums := []string{}
for k := range manifestData {
blobSums = append(blobSums, k)
}
manifest := struct {
Name string
BlobSums []string
History []map[string][]byte
}{
Name: remoteName,
BlobSums: blobSums,
History: []map[string][]byte{
manifestData,
},
}
manifestBuf, err := json.Marshal(manifest)
if err != nil {
return job.Error(err)
}
// Next, push the manifest
log.Debugf("SUCH MANIFEST %s:%s -- %s", localName, tag, manifestBuf)
err = r.PutV2ImageManifest(remoteName, tag, bytes.NewReader(manifestBuf), nil)
if err != nil {
return job.Error(err)
}
// we should wrap up this CmdPush here
return engine.StatusOK
}
if err != nil {
reposLen := 1
if tag == "" {
reposLen = len(s.Repositories[localName])
}
job.Stdout.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", localName, reposLen))
// If it fails, try to get the repository
if localRepo, exists := s.Repositories[localName]; exists {
if err := s.pushRepository(r, job.Stdout, localName, remoteName, localRepo, tag, sf); err != nil {
return job.Error(err)
}
return engine.StatusOK
}
return job.Error(err)
}
var token []string
job.Stdout.Write(sf.FormatStatus("", "The push refers to an image: [%s]", localName))
if _, err := s.pushImage(r, job.Stdout, remoteName, img.ID, endpoint.String(), token, sf); err != nil {
return job.Error(err)
}
return engine.StatusOK
}