diff --git a/directory/directory_src.go b/directory/directory_src.go index 8b509112a..4fc20b1ca 100644 --- a/directory/directory_src.go +++ b/directory/directory_src.go @@ -5,19 +5,33 @@ import ( "io" "os" + "github.com/containers/image/v5/internal/imagesource/impl" + "github.com/containers/image/v5/internal/imagesource/stubs" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" ) type dirImageSource struct { + impl.PropertyMethodsInitialize + impl.DoesNotAffectLayerInfosForCopy + stubs.NoGetBlobAtInitialize + ref dirReference } // newImageSource returns an ImageSource reading from an existing directory. // The caller must call .Close() on the returned ImageSource. -func newImageSource(ref dirReference) types.ImageSource { - return &dirImageSource{ref} +func newImageSource(ref dirReference) private.ImageSource { + return &dirImageSource{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + HasThreadSafeGetBlob: false, + }), + NoGetBlobAtInitialize: stubs.NoGetBlobAt(ref), + + ref: ref, + } } // Reference returns the reference used to set up this source, _as specified by the user_ @@ -43,11 +57,6 @@ func (s *dirImageSource) GetManifest(ctx context.Context, instanceDigest *digest return m, manifest.GuessMIMEType(m), err } -// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. -func (s *dirImageSource) HasThreadSafeGetBlob() bool { - return false -} - // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. @@ -81,15 +90,3 @@ func (s *dirImageSource) GetSignatures(ctx context.Context, instanceDigest *dige } return signatures, nil } - -// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer -// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() -// to read the image's layers. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). -// The Digest field is guaranteed to be provided; Size may be -1. -// WARNING: The list may contain duplicates, and they are semantically relevant. -func (s *dirImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { - return nil, nil -} diff --git a/directory/directory_test.go b/directory/directory_test.go index c75ebf13c..c9154561b 100644 --- a/directory/directory_test.go +++ b/directory/directory_test.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/require" ) +var _ private.ImageSource = (*dirImageSource)(nil) var _ private.ImageDestination = (*dirImageDestination)(nil) func TestDestinationReference(t *testing.T) { diff --git a/docker/archive/src.go b/docker/archive/src.go index 7acca210e..5604a5121 100644 --- a/docker/archive/src.go +++ b/docker/archive/src.go @@ -4,6 +4,7 @@ import ( "context" "github.com/containers/image/v5/docker/internal/tarfile" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/types" ) @@ -14,7 +15,7 @@ type archiveImageSource struct { // newImageSource returns a types.ImageSource for the specified image reference. // The caller must call .Close() on the returned ImageSource. -func newImageSource(ctx context.Context, sys *types.SystemContext, ref archiveReference) (types.ImageSource, error) { +func newImageSource(ctx context.Context, sys *types.SystemContext, ref archiveReference) (private.ImageSource, error) { var archive *tarfile.Reader var closeArchive bool if ref.archiveReader != nil { @@ -28,7 +29,7 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref archiveRe archive = a closeArchive = true } - src := tarfile.NewSource(archive, closeArchive, ref.ref, ref.sourceIndex) + src := tarfile.NewSource(archive, closeArchive, ref.Transport().Name(), ref.ref, ref.sourceIndex) return &archiveImageSource{ Source: src, ref: ref, diff --git a/docker/archive/src_test.go b/docker/archive/src_test.go new file mode 100644 index 000000000..b98abea76 --- /dev/null +++ b/docker/archive/src_test.go @@ -0,0 +1,5 @@ +package archive + +import "github.com/containers/image/v5/internal/private" + +var _ private.ImageSource = (*archiveImageSource)(nil) diff --git a/docker/daemon/daemon_src.go b/docker/daemon/daemon_src.go index d4b0b85c6..754e7ef2a 100644 --- a/docker/daemon/daemon_src.go +++ b/docker/daemon/daemon_src.go @@ -4,6 +4,7 @@ import ( "context" "github.com/containers/image/v5/docker/internal/tarfile" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/types" perrors "github.com/pkg/errors" ) @@ -22,7 +23,7 @@ type daemonImageSource struct { // (We could, perhaps, expect an exact sequence, assume that the first plaintext file // is the config, and that the following len(RootFS) files are the layers, but that feels // way too brittle.) -func newImageSource(ctx context.Context, sys *types.SystemContext, ref daemonReference) (types.ImageSource, error) { +func newImageSource(ctx context.Context, sys *types.SystemContext, ref daemonReference) (private.ImageSource, error) { c, err := newDockerClient(sys) if err != nil { return nil, perrors.Wrap(err, "initializing docker engine client") @@ -39,7 +40,7 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref daemonRef if err != nil { return nil, err } - src := tarfile.NewSource(archive, true, nil, -1) + src := tarfile.NewSource(archive, true, ref.Transport().Name(), nil, -1) return &daemonImageSource{ ref: ref, Source: src, diff --git a/docker/daemon/daemon_src_test.go b/docker/daemon/daemon_src_test.go new file mode 100644 index 000000000..7214ce701 --- /dev/null +++ b/docker/daemon/daemon_src_test.go @@ -0,0 +1,5 @@ +package daemon + +import "github.com/containers/image/v5/internal/private" + +var _ private.ImageSource = (*daemonImageSource)(nil) diff --git a/docker/docker_image_src.go b/docker/docker_image_src.go index bc0960aa0..3cb477167 100644 --- a/docker/docker_image_src.go +++ b/docker/docker_image_src.go @@ -16,6 +16,8 @@ import ( "sync" "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/internal/imagesource/impl" + "github.com/containers/image/v5/internal/imagesource/stubs" "github.com/containers/image/v5/internal/iolimits" "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/manifest" @@ -27,6 +29,10 @@ import ( ) type dockerImageSource struct { + impl.PropertyMethodsInitialize + impl.DoesNotAffectLayerInfosForCopy + stubs.ImplementsGetBlobAt + logicalRef dockerReference // The reference the user requested. physicalRef dockerReference // The actual reference we are accessing (possibly a mirror) c *dockerClient @@ -126,6 +132,10 @@ func newImageSourceAttempt(ctx context.Context, sys *types.SystemContext, logica client.tlsClientConfig.InsecureSkipVerify = pullSource.Endpoint.Insecure s := &dockerImageSource{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + HasThreadSafeGetBlob: true, + }), + logicalRef: logicalRef, physicalRef: physicalRef, c: client, @@ -148,23 +158,6 @@ func (s *dockerImageSource) Close() error { return nil } -// SupportsGetBlobAt() returns true if GetBlobAt (BlobChunkAccessor) is supported. -func (s *dockerImageSource) SupportsGetBlobAt() bool { - return true -} - -// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer -// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() -// to read the image's layers. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). -// The Digest field is guaranteed to be provided; Size may be -1. -// WARNING: The list may contain duplicates, and they are semantically relevant. -func (s *dockerImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { - return nil, nil -} - // simplifyContentType drops parameters from a HTTP media type (see https://tools.ietf.org/html/rfc7231#section-3.1.1.1) // Alternatively, an empty string is returned unchanged, and invalid values are "simplified" to an empty string. func simplifyContentType(contentType string) string { @@ -289,11 +282,6 @@ func getBlobSize(resp *http.Response) int64 { return size } -// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. -func (s *dockerImageSource) HasThreadSafeGetBlob() bool { - return true -} - // splitHTTP200ResponseToPartial splits a 200 response in multiple streams as specified by the chunks func splitHTTP200ResponseToPartial(streams chan io.ReadCloser, errs chan error, body io.ReadCloser, chunks []private.ImageSourceChunk) { defer close(streams) diff --git a/docker/internal/tarfile/src.go b/docker/internal/tarfile/src.go index cc7ad87bd..f5cfefe84 100644 --- a/docker/internal/tarfile/src.go +++ b/docker/internal/tarfile/src.go @@ -13,6 +13,8 @@ import ( "sync" "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/internal/imagesource/impl" + "github.com/containers/image/v5/internal/imagesource/stubs" "github.com/containers/image/v5/internal/iolimits" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/compression" @@ -23,6 +25,11 @@ import ( // Source is a partial implementation of types.ImageSource for reading from tarPath. type Source struct { + impl.PropertyMethodsInitialize + impl.NoSignatures + impl.DoesNotAffectLayerInfosForCopy + stubs.NoGetBlobAtInitialize + archive *Reader closeArchive bool // .Close() the archive when the source is closed. // If ref is nil and sourceIndex is -1, indicates the only image in the archive. @@ -48,8 +55,13 @@ type layerInfo struct { // NewSource returns a tarfile.Source for an image in the specified archive matching ref // and sourceIndex (or the only image if they are (nil, -1)). // The archive will be closed if closeArchive -func NewSource(archive *Reader, closeArchive bool, ref reference.NamedTagged, sourceIndex int) *Source { +func NewSource(archive *Reader, closeArchive bool, transportName string, ref reference.NamedTagged, sourceIndex int) *Source { return &Source{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + HasThreadSafeGetBlob: true, + }), + NoGetBlobAtInitialize: stubs.NoGetBlobAtRaw(transportName), + archive: archive, closeArchive: closeArchive, ref: ref, @@ -250,11 +262,6 @@ func (r uncompressedReadCloser) Close() error { return res } -// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. -func (s *Source) HasThreadSafeGetBlob() bool { - return true -} - // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. @@ -308,25 +315,3 @@ func (s *Source) GetBlob(ctx context.Context, info types.BlobInfo, cache types.B return nil, 0, fmt.Errorf("Unknown blob %s", info.Digest) } - -// GetSignatures returns the image's signatures. It may use a remote (= slow) service. -// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, -// as there can be no secondary manifests. -func (s *Source) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { - if instanceDigest != nil { - // How did we even get here? GetManifest(ctx, nil) has returned a manifest.DockerV2Schema2MediaType. - return nil, errors.New(`Manifest lists are not supported by "docker-daemon:"`) - } - return [][]byte{}, nil -} - -// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer -// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() -// to read the image's layers. -// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, -// as the primary manifest can not be a list, so there can be no secondary manifests. -// The Digest field is guaranteed to be provided; Size may be -1. -// WARNING: The list may contain duplicates, and they are semantically relevant. -func (s *Source) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { - return nil, nil -} diff --git a/docker/internal/tarfile/src_test.go b/docker/internal/tarfile/src_test.go index c46eed641..1f8248ac5 100644 --- a/docker/internal/tarfile/src_test.go +++ b/docker/internal/tarfile/src_test.go @@ -47,7 +47,7 @@ func TestSourcePrepareLayerData(t *testing.T) { reader, err := NewReaderFromStream(nil, &tarfileBuffer) require.NoError(t, err, c.config) - src := NewSource(reader, true, nil, -1) + src := NewSource(reader, true, "transport name", nil, -1) require.NoError(t, err, c.config) defer src.Close() configStream, _, err := src.GetBlob(ctx, types.BlobInfo{ diff --git a/docker/tarfile/src.go b/docker/tarfile/src.go index ee341eb39..067716ce5 100644 --- a/docker/tarfile/src.go +++ b/docker/tarfile/src.go @@ -28,7 +28,7 @@ func NewSourceFromFileWithContext(sys *types.SystemContext, path string) (*Sourc if err != nil { return nil, err } - src := internal.NewSource(archive, true, nil, -1) + src := internal.NewSource(archive, true, "[An external docker/tarfile caller]", nil, -1) return &Source{internal: src}, nil } @@ -49,7 +49,7 @@ func NewSourceFromStreamWithSystemContext(sys *types.SystemContext, inputStream if err != nil { return nil, err } - src := internal.NewSource(archive, true, nil, -1) + src := internal.NewSource(archive, true, "[An external docker/tarfile caller]", nil, -1) return &Source{internal: src}, nil } diff --git a/internal/imagesource/impl/layer_infos.go b/internal/imagesource/impl/layer_infos.go new file mode 100644 index 000000000..d5eae6351 --- /dev/null +++ b/internal/imagesource/impl/layer_infos.go @@ -0,0 +1,23 @@ +package impl + +import ( + "context" + + "github.com/containers/image/v5/types" + "github.com/opencontainers/go-digest" +) + +// DoesNotAffectLayerInfosForCopy implements LayerInfosForCopy() that returns nothing. +type DoesNotAffectLayerInfosForCopy struct{} + +// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer +// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() +// to read the image's layers. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (stub DoesNotAffectLayerInfosForCopy) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { + return nil, nil +} diff --git a/internal/imagesource/impl/properties.go b/internal/imagesource/impl/properties.go new file mode 100644 index 000000000..73e8c78e9 --- /dev/null +++ b/internal/imagesource/impl/properties.go @@ -0,0 +1,27 @@ +package impl + +// Properties collects properties of an ImageSource that are constant throughout its lifetime +// (but might differ across instances). +type Properties struct { + // HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. + HasThreadSafeGetBlob bool +} + +// PropertyMethodsInitialize implements parts of private.ImageSource corresponding to Properties. +type PropertyMethodsInitialize struct { + // We need two separate structs, PropertyMethodsInitialize and Properties, because Go prohibits fields and methods with the same name. + + vals Properties +} + +// PropertyMethods creates an PropertyMethodsInitialize for vals. +func PropertyMethods(vals Properties) PropertyMethodsInitialize { + return PropertyMethodsInitialize{ + vals: vals, + } +} + +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (o PropertyMethodsInitialize) HasThreadSafeGetBlob() bool { + return o.vals.HasThreadSafeGetBlob +} diff --git a/internal/imagesource/impl/signatures.go b/internal/imagesource/impl/signatures.go new file mode 100644 index 000000000..ee25152b5 --- /dev/null +++ b/internal/imagesource/impl/signatures.go @@ -0,0 +1,18 @@ +package impl + +import ( + "context" + + "github.com/opencontainers/go-digest" +) + +// NoSignatures implements GetSignatures() that returns nothing. +type NoSignatures struct{} + +// GetSignatures returns the image's signatures. It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +func (stub NoSignatures) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + return nil, nil +} diff --git a/internal/imagesource/stubs/get_blob_at.go b/internal/imagesource/stubs/get_blob_at.go new file mode 100644 index 000000000..15aee6d42 --- /dev/null +++ b/internal/imagesource/stubs/get_blob_at.go @@ -0,0 +1,52 @@ +package stubs + +import ( + "context" + "fmt" + "io" + + "github.com/containers/image/v5/internal/private" + "github.com/containers/image/v5/types" +) + +// NoGetBlobAtInitialize implements parts of private.ImageSource +// for transports that don’t support GetBlobAt(). +// See NoGetBlobAt() below. +type NoGetBlobAtInitialize struct { + transportName string +} + +// NoGetBlobAt() creates a NoGetBlobAtInitialize for ref. +func NoGetBlobAt(ref types.ImageReference) NoGetBlobAtInitialize { + return NoGetBlobAtRaw(ref.Transport().Name()) +} + +// NoGetBlobAtRaw is the same thing as NoGetBlobAt, but it can be used +// in situations where no ImageReference is available. +func NoGetBlobAtRaw(transportName string) NoGetBlobAtInitialize { + return NoGetBlobAtInitialize{ + transportName: transportName, + } +} + +// SupportsGetBlobAt() returns true if GetBlobAt (BlobChunkAccessor) is supported. +func (stub NoGetBlobAtInitialize) SupportsGetBlobAt() bool { + return false +} + +// GetBlobAt returns a sequential channel of readers that contain data for the requested +// blob chunks, and a channel that might get a single error value. +// The specified chunks must be not overlapping and sorted by their offset. +// The readers must be fully consumed, in the order they are returned, before blocking +// to read the next chunk. +func (stub NoGetBlobAtInitialize) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) { + return nil, nil, fmt.Errorf("internal error: GetBlobAt is not supported by the %q transport", stub.transportName) +} + +// ImplementsGetBlobAt implements SupportsGetBlobAt() that returns true. +type ImplementsGetBlobAt struct{} + +// SupportsGetBlobAt() returns true if GetBlobAt (BlobChunkAccessor) is supported. +func (stub ImplementsGetBlobAt) SupportsGetBlobAt() bool { + return true +} diff --git a/internal/imagesource/stubs/stubs.go b/internal/imagesource/stubs/stubs.go new file mode 100644 index 000000000..134fd1b53 --- /dev/null +++ b/internal/imagesource/stubs/stubs.go @@ -0,0 +1,25 @@ +// Package stubs contains trivial stubs for parts of private.ImageSource. +// It can be used from internal/wrapper, so it should not drag in any extra dependencies. +// Compare with imagesource/impl, which might require non-trivial implementation work. +// +// There are two kinds of stubs: +// - Pure stubs, like ImplementsGetBlobAt. Those can just be included in an ImageSource +// implementation: +// +// type yourSource struct { +// stubs.ImplementsGetBlobAt +// … +// } +// - Stubs with a constructor, like NoGetBlobAtInitialize. The Initialize marker +// means that a constructor must be called: +// type yourSource struct { +// stubs.NoGetBlobAtInitialize +// … +// } +// +// dest := &yourSource{ +// … +// NoGetBlobAtInitialize: stubs.NoGetBlobAt(ref), +// } +// +package stubs diff --git a/internal/imagesource/wrapper.go b/internal/imagesource/wrapper.go index fe1be8d9e..deacc713d 100644 --- a/internal/imagesource/wrapper.go +++ b/internal/imagesource/wrapper.go @@ -1,14 +1,19 @@ package imagesource import ( - "context" - "fmt" - "io" - + "github.com/containers/image/v5/internal/imagesource/stubs" "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/types" ) +// wrapped provides the private.ImageSource operations +// for a source that only implements types.ImageSource +type wrapped struct { + stubs.NoGetBlobAtInitialize + + types.ImageSource +} + // FromPublic(src) returns an object that provides the private.ImageSource API // // Eventually, we might want to expose this function, and methods of the returned object, @@ -23,25 +28,9 @@ func FromPublic(src types.ImageSource) private.ImageSource { if src2, ok := src.(private.ImageSource); ok { return src2 } - return &wrapped{ImageSource: src} -} + return &wrapped{ + NoGetBlobAtInitialize: stubs.NoGetBlobAt(src.Reference()), -// wrapped provides the private.ImageSource operations -// for a source that only implements types.ImageSource -type wrapped struct { - types.ImageSource -} - -// SupportsGetBlobAt() returns true if GetBlobAt (BlobChunkAccessor) is supported. -func (w *wrapped) SupportsGetBlobAt() bool { - return false -} - -// GetBlobAt returns a sequential channel of readers that contain data for the requested -// blob chunks, and a channel that might get a single error value. -// The specified chunks must be not overlapping and sorted by their offset. -// The readers must be fully consumed, in the order they are returned, before blocking -// to read the next chunk. -func (w *wrapped) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) { - return nil, nil, fmt.Errorf("internal error: GetBlobAt is not supported by the %q transport", w.Reference().Transport().Name()) + ImageSource: src, + } } diff --git a/oci/archive/oci_src.go b/oci/archive/oci_src.go index d3ab04bc1..d53f2ae15 100644 --- a/oci/archive/oci_src.go +++ b/oci/archive/oci_src.go @@ -5,6 +5,8 @@ import ( "errors" "io" + "github.com/containers/image/v5/internal/imagesource" + "github.com/containers/image/v5/internal/private" ocilayout "github.com/containers/image/v5/oci/layout" "github.com/containers/image/v5/types" digest "github.com/opencontainers/go-digest" @@ -15,13 +17,13 @@ import ( type ociArchiveImageSource struct { ref ociArchiveReference - unpackedSrc types.ImageSource + unpackedSrc private.ImageSource tempDirRef tempDirOCIRef } // newImageSource returns an ImageSource for reading from an existing directory. // newImageSource untars the file and saves it in a temp directory -func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageSource, error) { +func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (private.ImageSource, error) { tempDirRef, err := createUntarTempDir(sys, ref) if err != nil { return nil, perrors.Wrap(err, "creating temp directory") @@ -34,9 +36,11 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiv } return nil, err } - return &ociArchiveImageSource{ref: ref, - unpackedSrc: unpackedSrc, - tempDirRef: tempDirRef}, nil + return &ociArchiveImageSource{ + ref: ref, + unpackedSrc: imagesource.FromPublic(unpackedSrc), + tempDirRef: tempDirRef, + }, nil } // LoadManifestDescriptor loads the manifest @@ -102,6 +106,20 @@ func (s *ociArchiveImageSource) GetBlob(ctx context.Context, info types.BlobInfo return s.unpackedSrc.GetBlob(ctx, info, cache) } +// SupportsGetBlobAt() returns true if GetBlobAt (BlobChunkAccessor) is supported. +func (s *ociArchiveImageSource) SupportsGetBlobAt() bool { + return s.unpackedSrc.SupportsGetBlobAt() +} + +// GetBlobAt returns a sequential channel of readers that contain data for the requested +// blob chunks, and a channel that might get a single error value. +// The specified chunks must be not overlapping and sorted by their offset. +// The readers must be fully consumed, in the order they are returned, before blocking +// to read the next chunk. +func (s *ociArchiveImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) { + return s.unpackedSrc.GetBlobAt(ctx, info, chunks) +} + // GetSignatures returns the image's signatures. It may use a remote (= slow) service. // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list diff --git a/oci/archive/oci_src_test.go b/oci/archive/oci_src_test.go new file mode 100644 index 000000000..6f00afc0c --- /dev/null +++ b/oci/archive/oci_src_test.go @@ -0,0 +1,5 @@ +package archive + +import "github.com/containers/image/v5/internal/private" + +var _ private.ImageSource = (*ociArchiveImageSource)(nil) diff --git a/oci/layout/oci_src.go b/oci/layout/oci_src.go index b78c23565..db627015c 100644 --- a/oci/layout/oci_src.go +++ b/oci/layout/oci_src.go @@ -9,6 +9,9 @@ import ( "os" "strconv" + "github.com/containers/image/v5/internal/imagesource/impl" + "github.com/containers/image/v5/internal/imagesource/stubs" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/tlsclientconfig" "github.com/containers/image/v5/types" @@ -19,6 +22,11 @@ import ( ) type ociImageSource struct { + impl.PropertyMethodsInitialize + impl.NoSignatures + impl.DoesNotAffectLayerInfosForCopy + stubs.NoGetBlobAtInitialize + ref ociReference index *imgspecv1.Index descriptor imgspecv1.Descriptor @@ -27,7 +35,7 @@ type ociImageSource struct { } // newImageSource returns an ImageSource for reading from an existing directory. -func newImageSource(sys *types.SystemContext, ref ociReference) (types.ImageSource, error) { +func newImageSource(sys *types.SystemContext, ref ociReference) (private.ImageSource, error) { tr := tlsclientconfig.NewTransport() tr.TLSClientConfig = tlsconfig.ServerDefault() @@ -48,7 +56,17 @@ func newImageSource(sys *types.SystemContext, ref ociReference) (types.ImageSour if err != nil { return nil, err } - d := &ociImageSource{ref: ref, index: index, descriptor: descriptor, client: client} + d := &ociImageSource{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + HasThreadSafeGetBlob: false, + }), + NoGetBlobAtInitialize: stubs.NoGetBlobAt(ref), + + ref: ref, + index: index, + descriptor: descriptor, + client: client, + } if sys != nil { // TODO(jonboulle): check dir existence? d.sharedBlobDir = sys.OCISharedBlobDirPath @@ -104,11 +122,6 @@ func (s *ociImageSource) GetManifest(ctx context.Context, instanceDigest *digest return m, mimeType, nil } -// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. -func (s *ociImageSource) HasThreadSafeGetBlob() bool { - return false -} - // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. @@ -138,14 +151,6 @@ func (s *ociImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache return r, fi.Size(), nil } -// GetSignatures returns the image's signatures. It may use a remote (= slow) service. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). -func (s *ociImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { - return [][]byte{}, nil -} - // getExternalBlob returns the reader of the first available blob URL from urls, which must not be empty. // This function can return nil reader when no url is supported by this function. In this case, the caller // should fallback to fetch the non-external blob (i.e. pull from the registry). @@ -188,18 +193,6 @@ func (s *ociImageSource) getExternalBlob(ctx context.Context, urls []string) (io return nil, 0, errWrap } -// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer -// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() -// to read the image's layers. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). -// The Digest field is guaranteed to be provided; Size may be -1. -// WARNING: The list may contain duplicates, and they are semantically relevant. -func (s *ociImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { - return nil, nil -} - func getBlobSize(resp *http.Response) int64 { size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) if err != nil { diff --git a/oci/layout/oci_src_test.go b/oci/layout/oci_src_test.go index 025d1555d..b52e64e38 100644 --- a/oci/layout/oci_src_test.go +++ b/oci/layout/oci_src_test.go @@ -12,6 +12,7 @@ import ( "strings" "testing" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/pkg/blobinfocache/memory" "github.com/containers/image/v5/types" digest "github.com/opencontainers/go-digest" @@ -19,6 +20,8 @@ import ( "github.com/stretchr/testify/require" ) +var _ private.ImageSource = (*ociImageSource)(nil) + const RemoteLayerContent = "This is the remote layer content" var httpServerAddr string diff --git a/openshift/openshift.go b/openshift/openshift.go index 6147b9c0d..b2e4dfd9e 100644 --- a/openshift/openshift.go +++ b/openshift/openshift.go @@ -3,7 +3,6 @@ package openshift import ( "bytes" "context" - "crypto/rand" "encoding/json" "errors" "fmt" @@ -12,19 +11,9 @@ import ( "net/url" "strings" - "github.com/containers/image/v5/docker" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/internal/blobinfocache" - "github.com/containers/image/v5/internal/imagedestination" - "github.com/containers/image/v5/internal/imagedestination/impl" - "github.com/containers/image/v5/internal/imagedestination/stubs" "github.com/containers/image/v5/internal/iolimits" - "github.com/containers/image/v5/internal/private" - "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/types" "github.com/containers/image/v5/version" - "github.com/opencontainers/go-digest" - perrors "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -164,374 +153,6 @@ func (c *openshiftClient) convertDockerImageReference(ref string) (string, error return reference.Domain(c.ref.dockerReference) + "/" + parts[1], nil } -type openshiftImageSource struct { - client *openshiftClient - // Values specific to this image - sys *types.SystemContext - // State - docker types.ImageSource // The docker/distribution API endpoint, or nil if not resolved yet - imageStreamImageName string // Resolved image identifier, or "" if not known yet -} - -// newImageSource creates a new ImageSource for the specified reference. -// The caller must call .Close() on the returned ImageSource. -func newImageSource(sys *types.SystemContext, ref openshiftReference) (types.ImageSource, error) { - client, err := newOpenshiftClient(ref) - if err != nil { - return nil, err - } - - return &openshiftImageSource{ - client: client, - sys: sys, - }, nil -} - -// Reference returns the reference used to set up this source, _as specified by the user_ -// (not as the image itself, or its underlying storage, claims). This can be used e.g. to determine which public keys are trusted for this image. -func (s *openshiftImageSource) Reference() types.ImageReference { - return s.client.ref -} - -// Close removes resources associated with an initialized ImageSource, if any. -func (s *openshiftImageSource) Close() error { - if s.docker != nil { - err := s.docker.Close() - s.docker = nil - - return err - } - - return nil -} - -// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). -// It may use a remote (= slow) service. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); -// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). -func (s *openshiftImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { - if err := s.ensureImageIsResolved(ctx); err != nil { - return nil, "", err - } - return s.docker.GetManifest(ctx, instanceDigest) -} - -// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. -func (s *openshiftImageSource) HasThreadSafeGetBlob() bool { - return false -} - -// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). -// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. -// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. -func (s *openshiftImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) { - if err := s.ensureImageIsResolved(ctx); err != nil { - return nil, 0, err - } - return s.docker.GetBlob(ctx, info, cache) -} - -// GetSignatures returns the image's signatures. It may use a remote (= slow) service. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). -func (s *openshiftImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { - var imageStreamImageName string - if instanceDigest == nil { - if err := s.ensureImageIsResolved(ctx); err != nil { - return nil, err - } - imageStreamImageName = s.imageStreamImageName - } else { - imageStreamImageName = instanceDigest.String() - } - image, err := s.client.getImage(ctx, imageStreamImageName) - if err != nil { - return nil, err - } - var sigs [][]byte - for _, sig := range image.Signatures { - if sig.Type == imageSignatureTypeAtomic { - sigs = append(sigs, sig.Content) - } - } - return sigs, nil -} - -// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer -// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() -// to read the image's layers. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). -// The Digest field is guaranteed to be provided; Size may be -1. -// WARNING: The list may contain duplicates, and they are semantically relevant. -func (s *openshiftImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { - return nil, nil -} - -// ensureImageIsResolved sets up s.docker and s.imageStreamImageName -func (s *openshiftImageSource) ensureImageIsResolved(ctx context.Context) error { - if s.docker != nil { - return nil - } - - // FIXME: validate components per validation.IsValidPathSegmentName? - path := fmt.Sprintf("/oapi/v1/namespaces/%s/imagestreams/%s", s.client.ref.namespace, s.client.ref.stream) - body, err := s.client.doRequest(ctx, http.MethodGet, path, nil) - if err != nil { - return err - } - // Note: This does absolutely no kind/version checking or conversions. - var is imageStream - if err := json.Unmarshal(body, &is); err != nil { - return err - } - var te *tagEvent - for _, tag := range is.Status.Tags { - if tag.Tag != s.client.ref.dockerReference.Tag() { - continue - } - if len(tag.Items) > 0 { - te = &tag.Items[0] - break - } - } - if te == nil { - return errors.New("No matching tag found") - } - logrus.Debugf("tag event %#v", te) - dockerRefString, err := s.client.convertDockerImageReference(te.DockerImageReference) - if err != nil { - return err - } - logrus.Debugf("Resolved reference %#v", dockerRefString) - dockerRef, err := docker.ParseReference("//" + dockerRefString) - if err != nil { - return err - } - d, err := dockerRef.NewImageSource(ctx, s.sys) - if err != nil { - return err - } - s.docker = d - s.imageStreamImageName = te.Image - return nil -} - -type openshiftImageDestination struct { - impl.Compat - stubs.AlwaysSupportsSignatures - - client *openshiftClient - docker private.ImageDestination // The docker/distribution API endpoint - // State - imageStreamImageName string // "" if not yet known -} - -// newImageDestination creates a new ImageDestination for the specified reference. -func newImageDestination(ctx context.Context, sys *types.SystemContext, ref openshiftReference) (private.ImageDestination, error) { - client, err := newOpenshiftClient(ref) - if err != nil { - return nil, err - } - - // FIXME: Should this always use a digest, not a tag? Uploading to Docker by tag requires the tag _inside_ the manifest to match, - // i.e. a single signed image cannot be available under multiple tags. But with types.ImageDestination, we don't know - // the manifest digest at this point. - dockerRefString := fmt.Sprintf("//%s/%s/%s:%s", reference.Domain(client.ref.dockerReference), client.ref.namespace, client.ref.stream, client.ref.dockerReference.Tag()) - dockerRef, err := docker.ParseReference(dockerRefString) - if err != nil { - return nil, err - } - docker, err := dockerRef.NewImageDestination(ctx, sys) - if err != nil { - return nil, err - } - - d := &openshiftImageDestination{ - client: client, - docker: imagedestination.FromPublic(docker), - } - d.Compat = impl.AddCompat(d) - return d, nil -} - -// Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, -// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects. -func (d *openshiftImageDestination) Reference() types.ImageReference { - return d.client.ref -} - -// Close removes resources associated with an initialized ImageDestination, if any. -func (d *openshiftImageDestination) Close() error { - return d.docker.Close() -} - -func (d *openshiftImageDestination) SupportedManifestMIMETypes() []string { - return d.docker.SupportedManifestMIMETypes() -} - -func (d *openshiftImageDestination) DesiredLayerCompression() types.LayerCompression { - return types.Compress -} - -// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually -// uploaded to the image destination, true otherwise. -func (d *openshiftImageDestination) AcceptsForeignLayerURLs() bool { - return true -} - -// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise. -func (d *openshiftImageDestination) MustMatchRuntimeOS() bool { - return false -} - -// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(), -// and would prefer to receive an unmodified manifest instead of one modified for the destination. -// Does not make a difference if Reference().DockerReference() is nil. -func (d *openshiftImageDestination) IgnoresEmbeddedDockerReference() bool { - return d.docker.IgnoresEmbeddedDockerReference() -} - -// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. -func (d *openshiftImageDestination) HasThreadSafePutBlob() bool { - return false -} - -// SupportsPutBlobPartial returns true if PutBlobPartial is supported. -func (d *openshiftImageDestination) SupportsPutBlobPartial() bool { - return d.docker.SupportsPutBlobPartial() -} - -// PutBlobWithOptions writes contents of stream and returns data representing the result. -// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. -// inputInfo.Size is the expected length of stream, if known. -// inputInfo.MediaType describes the blob format, if known. -// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available -// to any other readers for download using the supplied digest. -// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (d *openshiftImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options private.PutBlobOptions) (types.BlobInfo, error) { - return d.docker.PutBlobWithOptions(ctx, stream, inputInfo, options) -} - -// PutBlobPartial attempts to create a blob using the data that is already present -// at the destination. chunkAccessor is accessed in a non-sequential way to retrieve the missing chunks. -// It is available only if SupportsPutBlobPartial(). -// Even if SupportsPutBlobPartial() returns true, the call can fail, in which case the caller -// should fall back to PutBlobWithOptions. -func (d *openshiftImageDestination) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, cache blobinfocache.BlobInfoCache2) (types.BlobInfo, error) { - return d.docker.PutBlobPartial(ctx, chunkAccessor, srcInfo, cache) -} - -// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination -// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). -// info.Digest must not be empty. -// If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may -// include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be -// reflected in the manifest that will be written. -// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. -func (d *openshiftImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { - return d.docker.TryReusingBlobWithOptions(ctx, info, options) -} - -// PutManifest writes manifest to the destination. -// FIXME? This should also receive a MIME type if known, to differentiate between schema versions. -// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), -// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. -func (d *openshiftImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { - if instanceDigest == nil { - manifestDigest, err := manifest.Digest(m) - if err != nil { - return err - } - d.imageStreamImageName = manifestDigest.String() - } - return d.docker.PutManifest(ctx, m, instanceDigest) -} - -func (d *openshiftImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { - var imageStreamImageName string - if instanceDigest == nil { - if d.imageStreamImageName == "" { - return errors.New("Internal error: Unknown manifest digest, can't add signatures") - } - imageStreamImageName = d.imageStreamImageName - } else { - imageStreamImageName = instanceDigest.String() - } - - // Because image signatures are a shared resource in Atomic Registry, the default upload - // always adds signatures. Eventually we should also allow removing signatures. - - if len(signatures) == 0 { - return nil // No need to even read the old state. - } - - image, err := d.client.getImage(ctx, imageStreamImageName) - if err != nil { - return err - } - existingSigNames := map[string]struct{}{} - for _, sig := range image.Signatures { - existingSigNames[sig.objectMeta.Name] = struct{}{} - } - -sigExists: - for _, newSig := range signatures { - for _, existingSig := range image.Signatures { - if existingSig.Type == imageSignatureTypeAtomic && bytes.Equal(existingSig.Content, newSig) { - continue sigExists - } - } - - // The API expect us to invent a new unique name. This is racy, but hopefully good enough. - var signatureName string - for { - randBytes := make([]byte, 16) - n, err := rand.Read(randBytes) - if err != nil || n != 16 { - return perrors.Wrapf(err, "generating random signature len %d", n) - } - signatureName = fmt.Sprintf("%s@%032x", imageStreamImageName, randBytes) - if _, ok := existingSigNames[signatureName]; !ok { - break - } - } - // Note: This does absolutely no kind/version checking or conversions. - sig := imageSignature{ - typeMeta: typeMeta{ - Kind: "ImageSignature", - APIVersion: "v1", - }, - objectMeta: objectMeta{Name: signatureName}, - Type: imageSignatureTypeAtomic, - Content: newSig, - } - body, err := json.Marshal(sig) - if err != nil { - return err - } - _, err = d.client.doRequest(ctx, http.MethodPost, "/oapi/v1/imagesignatures", body) - if err != nil { - return err - } - } - - return nil -} - -// Commit marks the process of storing the image as successful and asks for the image to be persisted. -// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list -// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the -// original manifest list digest, if desired. -// WARNING: This does not have any transactional semantics: -// - Uploaded data MAY be visible to others before Commit() is called -// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) -func (d *openshiftImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error { - return d.docker.Commit(ctx, unparsedToplevel) -} - // These structs are subsets of github.com/openshift/origin/pkg/image/api/v1 and its dependencies. type imageStream struct { Status imageStreamStatus `json:"status,omitempty"` diff --git a/openshift/openshift_dest.go b/openshift/openshift_dest.go new file mode 100644 index 000000000..db83e4e66 --- /dev/null +++ b/openshift/openshift_dest.go @@ -0,0 +1,237 @@ +package openshift + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + "github.com/containers/image/v5/docker" + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/internal/blobinfocache" + "github.com/containers/image/v5/internal/imagedestination" + "github.com/containers/image/v5/internal/imagedestination/impl" + "github.com/containers/image/v5/internal/imagedestination/stubs" + "github.com/containers/image/v5/internal/private" + "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/types" + "github.com/opencontainers/go-digest" + perrors "github.com/pkg/errors" +) + +type openshiftImageDestination struct { + impl.Compat + stubs.AlwaysSupportsSignatures + + client *openshiftClient + docker private.ImageDestination // The docker/distribution API endpoint + // State + imageStreamImageName string // "" if not yet known +} + +// newImageDestination creates a new ImageDestination for the specified reference. +func newImageDestination(ctx context.Context, sys *types.SystemContext, ref openshiftReference) (private.ImageDestination, error) { + client, err := newOpenshiftClient(ref) + if err != nil { + return nil, err + } + + // FIXME: Should this always use a digest, not a tag? Uploading to Docker by tag requires the tag _inside_ the manifest to match, + // i.e. a single signed image cannot be available under multiple tags. But with types.ImageDestination, we don't know + // the manifest digest at this point. + dockerRefString := fmt.Sprintf("//%s/%s/%s:%s", reference.Domain(client.ref.dockerReference), client.ref.namespace, client.ref.stream, client.ref.dockerReference.Tag()) + dockerRef, err := docker.ParseReference(dockerRefString) + if err != nil { + return nil, err + } + docker, err := dockerRef.NewImageDestination(ctx, sys) + if err != nil { + return nil, err + } + + d := &openshiftImageDestination{ + client: client, + docker: imagedestination.FromPublic(docker), + } + d.Compat = impl.AddCompat(d) + return d, nil +} + +// Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, +// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects. +func (d *openshiftImageDestination) Reference() types.ImageReference { + return d.client.ref +} + +// Close removes resources associated with an initialized ImageDestination, if any. +func (d *openshiftImageDestination) Close() error { + return d.docker.Close() +} + +func (d *openshiftImageDestination) SupportedManifestMIMETypes() []string { + return d.docker.SupportedManifestMIMETypes() +} + +func (d *openshiftImageDestination) DesiredLayerCompression() types.LayerCompression { + return types.Compress +} + +// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually +// uploaded to the image destination, true otherwise. +func (d *openshiftImageDestination) AcceptsForeignLayerURLs() bool { + return true +} + +// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise. +func (d *openshiftImageDestination) MustMatchRuntimeOS() bool { + return false +} + +// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(), +// and would prefer to receive an unmodified manifest instead of one modified for the destination. +// Does not make a difference if Reference().DockerReference() is nil. +func (d *openshiftImageDestination) IgnoresEmbeddedDockerReference() bool { + return d.docker.IgnoresEmbeddedDockerReference() +} + +// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. +func (d *openshiftImageDestination) HasThreadSafePutBlob() bool { + return false +} + +// SupportsPutBlobPartial returns true if PutBlobPartial is supported. +func (d *openshiftImageDestination) SupportsPutBlobPartial() bool { + return d.docker.SupportsPutBlobPartial() +} + +// PutBlobWithOptions writes contents of stream and returns data representing the result. +// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. +// inputInfo.Size is the expected length of stream, if known. +// inputInfo.MediaType describes the blob format, if known. +// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available +// to any other readers for download using the supplied digest. +// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. +func (d *openshiftImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options private.PutBlobOptions) (types.BlobInfo, error) { + return d.docker.PutBlobWithOptions(ctx, stream, inputInfo, options) +} + +// PutBlobPartial attempts to create a blob using the data that is already present +// at the destination. chunkAccessor is accessed in a non-sequential way to retrieve the missing chunks. +// It is available only if SupportsPutBlobPartial(). +// Even if SupportsPutBlobPartial() returns true, the call can fail, in which case the caller +// should fall back to PutBlobWithOptions. +func (d *openshiftImageDestination) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, cache blobinfocache.BlobInfoCache2) (types.BlobInfo, error) { + return d.docker.PutBlobPartial(ctx, chunkAccessor, srcInfo, cache) +} + +// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination +// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). +// info.Digest must not be empty. +// If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may +// include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be +// reflected in the manifest that will be written. +// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. +func (d *openshiftImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { + return d.docker.TryReusingBlobWithOptions(ctx, info, options) +} + +// PutManifest writes manifest to the destination. +// FIXME? This should also receive a MIME type if known, to differentiate between schema versions. +// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema), +// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError. +func (d *openshiftImageDestination) PutManifest(ctx context.Context, m []byte, instanceDigest *digest.Digest) error { + if instanceDigest == nil { + manifestDigest, err := manifest.Digest(m) + if err != nil { + return err + } + d.imageStreamImageName = manifestDigest.String() + } + return d.docker.PutManifest(ctx, m, instanceDigest) +} + +func (d *openshiftImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { + var imageStreamImageName string + if instanceDigest == nil { + if d.imageStreamImageName == "" { + return errors.New("Internal error: Unknown manifest digest, can't add signatures") + } + imageStreamImageName = d.imageStreamImageName + } else { + imageStreamImageName = instanceDigest.String() + } + + // Because image signatures are a shared resource in Atomic Registry, the default upload + // always adds signatures. Eventually we should also allow removing signatures. + + if len(signatures) == 0 { + return nil // No need to even read the old state. + } + + image, err := d.client.getImage(ctx, imageStreamImageName) + if err != nil { + return err + } + existingSigNames := map[string]struct{}{} + for _, sig := range image.Signatures { + existingSigNames[sig.objectMeta.Name] = struct{}{} + } + +sigExists: + for _, newSig := range signatures { + for _, existingSig := range image.Signatures { + if existingSig.Type == imageSignatureTypeAtomic && bytes.Equal(existingSig.Content, newSig) { + continue sigExists + } + } + + // The API expect us to invent a new unique name. This is racy, but hopefully good enough. + var signatureName string + for { + randBytes := make([]byte, 16) + n, err := rand.Read(randBytes) + if err != nil || n != 16 { + return perrors.Wrapf(err, "generating random signature len %d", n) + } + signatureName = fmt.Sprintf("%s@%032x", imageStreamImageName, randBytes) + if _, ok := existingSigNames[signatureName]; !ok { + break + } + } + // Note: This does absolutely no kind/version checking or conversions. + sig := imageSignature{ + typeMeta: typeMeta{ + Kind: "ImageSignature", + APIVersion: "v1", + }, + objectMeta: objectMeta{Name: signatureName}, + Type: imageSignatureTypeAtomic, + Content: newSig, + } + body, err := json.Marshal(sig) + if err != nil { + return err + } + _, err = d.client.doRequest(ctx, http.MethodPost, "/oapi/v1/imagesignatures", body) + if err != nil { + return err + } + } + + return nil +} + +// Commit marks the process of storing the image as successful and asks for the image to be persisted. +// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list +// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the +// original manifest list digest, if desired. +// WARNING: This does not have any transactional semantics: +// - Uploaded data MAY be visible to others before Commit() is called +// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) +func (d *openshiftImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error { + return d.docker.Commit(ctx, unparsedToplevel) +} diff --git a/openshift/openshift_test.go b/openshift/openshift_dest_test.go similarity index 100% rename from openshift/openshift_test.go rename to openshift/openshift_dest_test.go diff --git a/openshift/openshift_src.go b/openshift/openshift_src.go new file mode 100644 index 000000000..8555c05e0 --- /dev/null +++ b/openshift/openshift_src.go @@ -0,0 +1,169 @@ +package openshift + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + + "github.com/containers/image/v5/docker" + "github.com/containers/image/v5/internal/imagesource/impl" + "github.com/containers/image/v5/internal/imagesource/stubs" + "github.com/containers/image/v5/internal/private" + "github.com/containers/image/v5/types" + "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" +) + +type openshiftImageSource struct { + impl.DoesNotAffectLayerInfosForCopy + // This is slightly suboptimal. We could forward GetBlobAt(), but we need to call ensureImageIsResolved in SupportsGetBlobAt(), + // and that method doesn’t provide a context for timing out. That could actually be fixed (SupportsGetBlobAt is private and we + // can change it), but this is a deprecated transport anyway, so for now we just punt. + stubs.NoGetBlobAtInitialize + + client *openshiftClient + // Values specific to this image + sys *types.SystemContext + // State + docker types.ImageSource // The docker/distribution API endpoint, or nil if not resolved yet + imageStreamImageName string // Resolved image identifier, or "" if not known yet +} + +// newImageSource creates a new ImageSource for the specified reference. +// The caller must call .Close() on the returned ImageSource. +func newImageSource(sys *types.SystemContext, ref openshiftReference) (private.ImageSource, error) { + client, err := newOpenshiftClient(ref) + if err != nil { + return nil, err + } + + return &openshiftImageSource{ + NoGetBlobAtInitialize: stubs.NoGetBlobAt(ref), + + client: client, + sys: sys, + }, nil +} + +// Reference returns the reference used to set up this source, _as specified by the user_ +// (not as the image itself, or its underlying storage, claims). This can be used e.g. to determine which public keys are trusted for this image. +func (s *openshiftImageSource) Reference() types.ImageReference { + return s.client.ref +} + +// Close removes resources associated with an initialized ImageSource, if any. +func (s *openshiftImageSource) Close() error { + if s.docker != nil { + err := s.docker.Close() + s.docker = nil + + return err + } + + return nil +} + +// GetManifest returns the image's manifest along with its MIME type (which may be empty when it can't be determined but the manifest is available). +// It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list); +// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists). +func (s *openshiftImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) { + if err := s.ensureImageIsResolved(ctx); err != nil { + return nil, "", err + } + return s.docker.GetManifest(ctx, instanceDigest) +} + +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (s *openshiftImageSource) HasThreadSafeGetBlob() bool { + return false +} + +// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). +// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. +// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. +func (s *openshiftImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) { + if err := s.ensureImageIsResolved(ctx); err != nil { + return nil, 0, err + } + return s.docker.GetBlob(ctx, info, cache) +} + +// GetSignatures returns the image's signatures. It may use a remote (= slow) service. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for +// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list +// (e.g. if the source never returns manifest lists). +func (s *openshiftImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { + var imageStreamImageName string + if instanceDigest == nil { + if err := s.ensureImageIsResolved(ctx); err != nil { + return nil, err + } + imageStreamImageName = s.imageStreamImageName + } else { + imageStreamImageName = instanceDigest.String() + } + image, err := s.client.getImage(ctx, imageStreamImageName) + if err != nil { + return nil, err + } + var sigs [][]byte + for _, sig := range image.Signatures { + if sig.Type == imageSignatureTypeAtomic { + sigs = append(sigs, sig.Content) + } + } + return sigs, nil +} + +// ensureImageIsResolved sets up s.docker and s.imageStreamImageName +func (s *openshiftImageSource) ensureImageIsResolved(ctx context.Context) error { + if s.docker != nil { + return nil + } + + // FIXME: validate components per validation.IsValidPathSegmentName? + path := fmt.Sprintf("/oapi/v1/namespaces/%s/imagestreams/%s", s.client.ref.namespace, s.client.ref.stream) + body, err := s.client.doRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return err + } + // Note: This does absolutely no kind/version checking or conversions. + var is imageStream + if err := json.Unmarshal(body, &is); err != nil { + return err + } + var te *tagEvent + for _, tag := range is.Status.Tags { + if tag.Tag != s.client.ref.dockerReference.Tag() { + continue + } + if len(tag.Items) > 0 { + te = &tag.Items[0] + break + } + } + if te == nil { + return errors.New("No matching tag found") + } + logrus.Debugf("tag event %#v", te) + dockerRefString, err := s.client.convertDockerImageReference(te.DockerImageReference) + if err != nil { + return err + } + logrus.Debugf("Resolved reference %#v", dockerRefString) + dockerRef, err := docker.ParseReference("//" + dockerRefString) + if err != nil { + return err + } + d, err := dockerRef.NewImageSource(ctx, s.sys) + if err != nil { + return err + } + s.docker = d + s.imageStreamImageName = te.Image + return nil +} diff --git a/openshift/openshift_src_test.go b/openshift/openshift_src_test.go new file mode 100644 index 000000000..0f8889289 --- /dev/null +++ b/openshift/openshift_src_test.go @@ -0,0 +1,5 @@ +package openshift + +import "github.com/containers/image/v5/internal/private" + +var _ private.ImageSource = (*openshiftImageSource)(nil) diff --git a/ostree/ostree_src.go b/ostree/ostree_src.go index 4b4e7a0ea..29f4460ec 100644 --- a/ostree/ostree_src.go +++ b/ostree/ostree_src.go @@ -14,6 +14,9 @@ import ( "strings" "unsafe" + "github.com/containers/image/v5/internal/imagesource/impl" + "github.com/containers/image/v5/internal/imagesource/stubs" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/types" "github.com/containers/storage/pkg/ioutils" @@ -34,6 +37,9 @@ import ( import "C" type ostreeImageSource struct { + impl.PropertyMethodsInitialize + stubs.NoGetBlobAtInitialize + ref ostreeReference tmpDir string repo *C.struct_OstreeRepo @@ -42,8 +48,17 @@ type ostreeImageSource struct { } // newImageSource returns an ImageSource for reading from an existing directory. -func newImageSource(tmpDir string, ref ostreeReference) (types.ImageSource, error) { - return &ostreeImageSource{ref: ref, tmpDir: tmpDir, compressed: nil}, nil +func newImageSource(tmpDir string, ref ostreeReference) (private.ImageSource, error) { + return &ostreeImageSource{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + HasThreadSafeGetBlob: false, + }), + NoGetBlobAtInitialize: stubs.NoGetBlobAt(ref), + + ref: ref, + tmpDir: tmpDir, + compressed: nil, + }, nil } // Reference returns the reference used to set up this source. @@ -263,11 +278,6 @@ func (s *ostreeImageSource) readSingleFile(commit, path string) (io.ReadCloser, return getter.Get(path) } -// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. -func (s *ostreeImageSource) HasThreadSafeGetBlob() bool { - return false -} - // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. diff --git a/ostree/ostree_src_test.go b/ostree/ostree_src_test.go new file mode 100644 index 000000000..60c35e69a --- /dev/null +++ b/ostree/ostree_src_test.go @@ -0,0 +1,8 @@ +//go:build containers_image_ostree +// +build containers_image_ostree + +package ostree + +import "github.com/containers/image/v5/internal/private" + +var _ private.ImageSource = (*ostreeImageSource)(nil) diff --git a/sif/src.go b/sif/src.go index ccf125966..a90d70fcd 100644 --- a/sif/src.go +++ b/sif/src.go @@ -9,6 +9,9 @@ import ( "io" "os" + "github.com/containers/image/v5/internal/imagesource/impl" + "github.com/containers/image/v5/internal/imagesource/stubs" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/internal/tmpdir" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" @@ -19,6 +22,11 @@ import ( ) type sifImageSource struct { + impl.PropertyMethodsInitialize + impl.NoSignatures + impl.DoesNotAffectLayerInfosForCopy + stubs.NoGetBlobAtInitialize + ref sifReference workDir string layerDigest digest.Digest @@ -55,7 +63,7 @@ func getBlobInfo(path string) (digest.Digest, int64, error) { // newImageSource returns an ImageSource for reading from an existing directory. // newImageSource extracts SIF objects and saves them in a temp directory. -func newImageSource(ctx context.Context, sys *types.SystemContext, ref sifReference) (types.ImageSource, error) { +func newImageSource(ctx context.Context, sys *types.SystemContext, ref sifReference) (private.ImageSource, error) { sifImg, err := sif.LoadContainerFromPath(ref.file, sif.OptLoadWithFlag(os.O_RDONLY)) if err != nil { return nil, fmt.Errorf("loading SIF file: %w", err) @@ -137,6 +145,11 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref sifRefere succeeded = true return &sifImageSource{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + HasThreadSafeGetBlob: true, + }), + NoGetBlobAtInitialize: stubs.NoGetBlobAt(ref), + ref: ref, workDir: workDir, layerDigest: layerDigest, @@ -158,11 +171,6 @@ func (s *sifImageSource) Close() error { return os.RemoveAll(s.workDir) } -// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. -func (s *sifImageSource) HasThreadSafeGetBlob() bool { - return true -} - // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. @@ -191,26 +199,3 @@ func (s *sifImageSource) GetManifest(ctx context.Context, instanceDigest *digest } return s.manifest, imgspecv1.MediaTypeImageManifest, nil } - -// GetSignatures returns the image's signatures. It may use a remote (= slow) service. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). -func (s *sifImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { - if instanceDigest != nil { - return nil, errors.New("manifest lists are not supported by the sif transport") - } - return nil, nil -} - -// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer -// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() -// to read the image's layers. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). -// The Digest field is guaranteed to be provided; Size may be -1. -// WARNING: The list may contain duplicates, and they are semantically relevant. -func (s *sifImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { - return nil, nil -} diff --git a/sif/src_test.go b/sif/src_test.go new file mode 100644 index 000000000..8402e23b1 --- /dev/null +++ b/sif/src_test.go @@ -0,0 +1,5 @@ +package sif + +import "github.com/containers/image/v5/internal/private" + +var _ private.ImageSource = (*sifImageSource)(nil) diff --git a/storage/storage_dest.go b/storage/storage_dest.go new file mode 100644 index 000000000..943edab9d --- /dev/null +++ b/storage/storage_dest.go @@ -0,0 +1,917 @@ +//go:build !containers_image_storage_stub +// +build !containers_image_storage_stub + +package storage + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "sync/atomic" + + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/internal/blobinfocache" + "github.com/containers/image/v5/internal/imagedestination/impl" + "github.com/containers/image/v5/internal/imagedestination/stubs" + "github.com/containers/image/v5/internal/private" + "github.com/containers/image/v5/internal/putblobdigest" + "github.com/containers/image/v5/internal/tmpdir" + "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/pkg/blobinfocache/none" + "github.com/containers/image/v5/types" + "github.com/containers/storage" + graphdriver "github.com/containers/storage/drivers" + "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/chunked" + "github.com/containers/storage/pkg/ioutils" + digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + perrors "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +var ( + // ErrBlobDigestMismatch could potentially be returned when PutBlob() is given a blob + // with a digest-based name that doesn't match its contents. + // Deprecated: PutBlob() doesn't do this any more (it just accepts the caller’s value), + // and there is no known user of this error. + ErrBlobDigestMismatch = errors.New("blob digest mismatch") + // ErrBlobSizeMismatch is returned when PutBlob() is given a blob + // with an expected size that doesn't match the reader. + ErrBlobSizeMismatch = errors.New("blob size mismatch") +) + +type storageImageDestination struct { + impl.Compat + impl.PropertyMethodsInitialize + stubs.ImplementsPutBlobPartial + stubs.AlwaysSupportsSignatures + + imageRef storageReference + directory string // Temporary directory where we store blobs until Commit() time + nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs + manifest []byte // Manifest contents, temporary + manifestDigest digest.Digest // Valid if len(manifest) != 0 + signatures []byte // Signature contents, temporary + signatureses map[digest.Digest][]byte // Instance signature contents, temporary + SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice + SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // Sizes of each manifest's signature slice + + // A storage destination may be used concurrently. Accesses are + // serialized via a mutex. Please refer to the individual comments + // below for details. + lock sync.Mutex + // Mapping from layer (by index) to the associated ID in the storage. + // It's protected *implicitly* since `commitLayer()`, at any given + // time, can only be executed by *one* goroutine. Please refer to + // `queueOrCommit()` for further details on how the single-caller + // guarantee is implemented. + indexToStorageID map[int]*string + // All accesses to below data are protected by `lock` which is made + // *explicit* in the code. + blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs + fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes + filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them + currentIndex int // The index of the layer to be committed (i.e., lower indices have already been committed) + indexToPulledLayerInfo map[int]*manifest.LayerInfo // Mapping from layer (by index) to pulled down blob + blobAdditionalLayer map[digest.Digest]storage.AdditionalLayer // Mapping from layer blobsums to their corresponding additional layer + diffOutputs map[digest.Digest]*graphdriver.DriverWithDifferOutput // Mapping from digest to differ output +} + +// newImageDestination sets us up to write a new image, caching blobs in a temporary directory until +// it's time to Commit() the image +func newImageDestination(sys *types.SystemContext, imageRef storageReference) (*storageImageDestination, error) { + directory, err := os.MkdirTemp(tmpdir.TemporaryDirectoryForBigFiles(sys), "storage") + if err != nil { + return nil, perrors.Wrapf(err, "creating a temporary directory") + } + dest := &storageImageDestination{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + SupportedManifestMIMETypes: []string{ + imgspecv1.MediaTypeImageManifest, + manifest.DockerV2Schema2MediaType, + manifest.DockerV2Schema1SignedMediaType, + manifest.DockerV2Schema1MediaType, + }, + // We ultimately have to decompress layers to populate trees on disk + // and need to explicitly ask for it here, so that the layers' MIME + // types can be set accordingly. + DesiredLayerCompression: types.PreserveOriginal, + AcceptsForeignLayerURLs: false, + MustMatchRuntimeOS: true, + IgnoresEmbeddedDockerReference: true, // Yes, we want the unmodified manifest + HasThreadSafePutBlob: true, + }), + + imageRef: imageRef, + directory: directory, + signatureses: make(map[digest.Digest][]byte), + blobDiffIDs: make(map[digest.Digest]digest.Digest), + blobAdditionalLayer: make(map[digest.Digest]storage.AdditionalLayer), + fileSizes: make(map[digest.Digest]int64), + filenames: make(map[digest.Digest]string), + SignatureSizes: []int{}, + SignaturesSizes: make(map[digest.Digest][]int), + indexToStorageID: make(map[int]*string), + indexToPulledLayerInfo: make(map[int]*manifest.LayerInfo), + diffOutputs: make(map[digest.Digest]*graphdriver.DriverWithDifferOutput), + } + dest.Compat = impl.AddCompat(dest) + return dest, nil +} + +// Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, +// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects. +func (s *storageImageDestination) Reference() types.ImageReference { + return s.imageRef +} + +// Close cleans up the temporary directory and additional layer store handlers. +func (s *storageImageDestination) Close() error { + for _, al := range s.blobAdditionalLayer { + al.Release() + } + for _, v := range s.diffOutputs { + if v.Target != "" { + _ = s.imageRef.transport.store.CleanupStagingDirectory(v.Target) + } + } + return os.RemoveAll(s.directory) +} + +func (s *storageImageDestination) computeNextBlobCacheFile() string { + return filepath.Join(s.directory, fmt.Sprintf("%d", atomic.AddInt32(&s.nextTempFileID, 1))) +} + +// PutBlobWithOptions writes contents of stream and returns data representing the result. +// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. +// inputInfo.Size is the expected length of stream, if known. +// inputInfo.MediaType describes the blob format, if known. +// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available +// to any other readers for download using the supplied digest. +// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. +func (s *storageImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, options private.PutBlobOptions) (types.BlobInfo, error) { + info, err := s.putBlobToPendingFile(ctx, stream, blobinfo, &options) + if err != nil { + return info, err + } + + if options.IsConfig || options.LayerIndex == nil { + return info, nil + } + + return info, s.queueOrCommit(ctx, info, *options.LayerIndex, options.EmptyLayer) +} + +// putBlobToPendingFile implements ImageDestination.PutBlobWithOptions, storing stream into an on-disk file. +// The caller must arrange the blob to be eventually committed using s.commitLayer(). +func (s *storageImageDestination) putBlobToPendingFile(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, options *private.PutBlobOptions) (types.BlobInfo, error) { + // Stores a layer or data blob in our temporary directory, checking that any information + // in the blobinfo matches the incoming data. + errorBlobInfo := types.BlobInfo{ + Digest: "", + Size: -1, + } + if blobinfo.Digest != "" { + if err := blobinfo.Digest.Validate(); err != nil { + return errorBlobInfo, fmt.Errorf("invalid digest %#v: %w", blobinfo.Digest.String(), err) + } + } + + // Set up to digest the blob if necessary, and count its size while saving it to a file. + filename := s.computeNextBlobCacheFile() + file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600) + if err != nil { + return errorBlobInfo, perrors.Wrapf(err, "creating temporary file %q", filename) + } + defer file.Close() + counter := ioutils.NewWriteCounter(file) + stream = io.TeeReader(stream, counter) + digester, stream := putblobdigest.DigestIfUnknown(stream, blobinfo) + decompressed, err := archive.DecompressStream(stream) + if err != nil { + return errorBlobInfo, perrors.Wrap(err, "setting up to decompress blob") + } + + diffID := digest.Canonical.Digester() + // Copy the data to the file. + // TODO: This can take quite some time, and should ideally be cancellable using ctx.Done(). + _, err = io.Copy(diffID.Hash(), decompressed) + decompressed.Close() + if err != nil { + return errorBlobInfo, perrors.Wrapf(err, "storing blob to file %q", filename) + } + + // Determine blob properties, and fail if information that we were given about the blob + // is known to be incorrect. + blobDigest := digester.Digest() + blobSize := blobinfo.Size + if blobSize < 0 { + blobSize = counter.Count + } else if blobinfo.Size != counter.Count { + return errorBlobInfo, ErrBlobSizeMismatch + } + + // Record information about the blob. + s.lock.Lock() + s.blobDiffIDs[blobDigest] = diffID.Digest() + s.fileSizes[blobDigest] = counter.Count + s.filenames[blobDigest] = filename + s.lock.Unlock() + // This is safe because we have just computed diffID, and blobDigest was either computed + // by us, or validated by the caller (usually copy.digestingReader). + options.Cache.RecordDigestUncompressedPair(blobDigest, diffID.Digest()) + return types.BlobInfo{ + Digest: blobDigest, + Size: blobSize, + MediaType: blobinfo.MediaType, + }, nil +} + +type zstdFetcher struct { + chunkAccessor private.BlobChunkAccessor + ctx context.Context + blobInfo types.BlobInfo +} + +// GetBlobAt converts from chunked.GetBlobAt to BlobChunkAccessor.GetBlobAt. +func (f *zstdFetcher) GetBlobAt(chunks []chunked.ImageSourceChunk) (chan io.ReadCloser, chan error, error) { + var newChunks []private.ImageSourceChunk + for _, v := range chunks { + i := private.ImageSourceChunk{ + Offset: v.Offset, + Length: v.Length, + } + newChunks = append(newChunks, i) + } + rc, errs, err := f.chunkAccessor.GetBlobAt(f.ctx, f.blobInfo, newChunks) + if _, ok := err.(private.BadPartialRequestError); ok { + err = chunked.ErrBadRequest{} + } + return rc, errs, err + +} + +// PutBlobPartial attempts to create a blob using the data that is already present +// at the destination. chunkAccessor is accessed in a non-sequential way to retrieve the missing chunks. +// It is available only if SupportsPutBlobPartial(). +// Even if SupportsPutBlobPartial() returns true, the call can fail, in which case the caller +// should fall back to PutBlobWithOptions. +func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, cache blobinfocache.BlobInfoCache2) (types.BlobInfo, error) { + fetcher := zstdFetcher{ + chunkAccessor: chunkAccessor, + ctx: ctx, + blobInfo: srcInfo, + } + + differ, err := chunked.GetDiffer(ctx, s.imageRef.transport.store, srcInfo.Size, srcInfo.Annotations, &fetcher) + if err != nil { + return srcInfo, err + } + + out, err := s.imageRef.transport.store.ApplyDiffWithDiffer("", nil, differ) + if err != nil { + return srcInfo, err + } + + blobDigest := srcInfo.Digest + + s.lock.Lock() + s.blobDiffIDs[blobDigest] = blobDigest + s.fileSizes[blobDigest] = 0 + s.filenames[blobDigest] = "" + s.diffOutputs[blobDigest] = out + s.lock.Unlock() + + return srcInfo, nil +} + +// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination +// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). +// info.Digest must not be empty. +// If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may +// include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be +// reflected in the manifest that will be written. +// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. +func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context, blobinfo types.BlobInfo, options private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { + reused, info, err := s.tryReusingBlobAsPending(ctx, blobinfo, &options) + if err != nil || !reused || options.LayerIndex == nil { + return reused, info, err + } + + return reused, info, s.queueOrCommit(ctx, info, *options.LayerIndex, options.EmptyLayer) +} + +// tryReusingBlobAsPending implements TryReusingBlobWithOptions, filling s.blobDiffIDs and other metadata. +// The caller must arrange the blob to be eventually committed using s.commitLayer(). +func (s *storageImageDestination) tryReusingBlobAsPending(ctx context.Context, blobinfo types.BlobInfo, options *private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { + // lock the entire method as it executes fairly quickly + s.lock.Lock() + defer s.lock.Unlock() + + if options.SrcRef != nil { + // Check if we have the layer in the underlying additional layer store. + aLayer, err := s.imageRef.transport.store.LookupAdditionalLayer(blobinfo.Digest, options.SrcRef.String()) + if err != nil && !errors.Is(err, storage.ErrLayerUnknown) { + return false, types.BlobInfo{}, perrors.Wrapf(err, `looking for compressed layers with digest %q and labels`, blobinfo.Digest) + } else if err == nil { + // Record the uncompressed value so that we can use it to calculate layer IDs. + s.blobDiffIDs[blobinfo.Digest] = aLayer.UncompressedDigest() + s.blobAdditionalLayer[blobinfo.Digest] = aLayer + return true, types.BlobInfo{ + Digest: blobinfo.Digest, + Size: aLayer.CompressedSize(), + MediaType: blobinfo.MediaType, + }, nil + } + } + + if blobinfo.Digest == "" { + return false, types.BlobInfo{}, errors.New(`Can not check for a blob with unknown digest`) + } + if err := blobinfo.Digest.Validate(); err != nil { + return false, types.BlobInfo{}, perrors.Wrapf(err, `Can not check for a blob with invalid digest`) + } + + // Check if we've already cached it in a file. + if size, ok := s.fileSizes[blobinfo.Digest]; ok { + return true, types.BlobInfo{ + Digest: blobinfo.Digest, + Size: size, + MediaType: blobinfo.MediaType, + }, nil + } + + // Check if we have a wasn't-compressed layer in storage that's based on that blob. + layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(blobinfo.Digest) + if err != nil && !errors.Is(err, storage.ErrLayerUnknown) { + return false, types.BlobInfo{}, perrors.Wrapf(err, `looking for layers with digest %q`, blobinfo.Digest) + } + if len(layers) > 0 { + // Save this for completeness. + s.blobDiffIDs[blobinfo.Digest] = layers[0].UncompressedDigest + return true, types.BlobInfo{ + Digest: blobinfo.Digest, + Size: layers[0].UncompressedSize, + MediaType: blobinfo.MediaType, + }, nil + } + + // Check if we have a was-compressed layer in storage that's based on that blob. + layers, err = s.imageRef.transport.store.LayersByCompressedDigest(blobinfo.Digest) + if err != nil && !errors.Is(err, storage.ErrLayerUnknown) { + return false, types.BlobInfo{}, perrors.Wrapf(err, `looking for compressed layers with digest %q`, blobinfo.Digest) + } + if len(layers) > 0 { + // Record the uncompressed value so that we can use it to calculate layer IDs. + s.blobDiffIDs[blobinfo.Digest] = layers[0].UncompressedDigest + return true, types.BlobInfo{ + Digest: blobinfo.Digest, + Size: layers[0].CompressedSize, + MediaType: blobinfo.MediaType, + }, nil + } + + // Does the blob correspond to a known DiffID which we already have available? + // Because we must return the size, which is unknown for unavailable compressed blobs, the returned BlobInfo refers to the + // uncompressed layer, and that can happen only if options.CanSubstitute, or if the incoming manifest already specifies the size. + if options.CanSubstitute || blobinfo.Size != -1 { + if uncompressedDigest := options.Cache.UncompressedDigest(blobinfo.Digest); uncompressedDigest != "" && uncompressedDigest != blobinfo.Digest { + layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(uncompressedDigest) + if err != nil && !errors.Is(err, storage.ErrLayerUnknown) { + return false, types.BlobInfo{}, perrors.Wrapf(err, `looking for layers with digest %q`, uncompressedDigest) + } + if len(layers) > 0 { + if blobinfo.Size != -1 { + s.blobDiffIDs[blobinfo.Digest] = layers[0].UncompressedDigest + return true, blobinfo, nil + } + if !options.CanSubstitute { + return false, types.BlobInfo{}, fmt.Errorf("Internal error: options.CanSubstitute was expected to be true for blobInfo %v", blobinfo) + } + s.blobDiffIDs[uncompressedDigest] = layers[0].UncompressedDigest + return true, types.BlobInfo{ + Digest: uncompressedDigest, + Size: layers[0].UncompressedSize, + MediaType: blobinfo.MediaType, + }, nil + } + } + } + + // Nope, we don't have it. + return false, types.BlobInfo{}, nil +} + +// computeID computes a recommended image ID based on information we have so far. If +// the manifest is not of a type that we recognize, we return an empty value, indicating +// that since we don't have a recommendation, a random ID should be used if one needs +// to be allocated. +func (s *storageImageDestination) computeID(m manifest.Manifest) string { + // Build the diffID list. We need the decompressed sums that we've been calculating to + // fill in the DiffIDs. It's expected (but not enforced by us) that the number of + // diffIDs corresponds to the number of non-EmptyLayer entries in the history. + var diffIDs []digest.Digest + switch m := m.(type) { + case *manifest.Schema1: + // Build a list of the diffIDs we've generated for the non-throwaway FS layers, + // in reverse of the order in which they were originally listed. + for i, compat := range m.ExtractedV1Compatibility { + if compat.ThrowAway { + continue + } + blobSum := m.FSLayers[i].BlobSum + diffID, ok := s.blobDiffIDs[blobSum] + if !ok { + logrus.Infof("error looking up diffID for layer %q", blobSum.String()) + return "" + } + diffIDs = append([]digest.Digest{diffID}, diffIDs...) + } + case *manifest.Schema2, *manifest.OCI1: + // We know the ID calculation for these formats doesn't actually use the diffIDs, + // so we don't need to populate the diffID list. + default: + return "" + } + id, err := m.ImageID(diffIDs) + if err != nil { + return "" + } + return id +} + +// getConfigBlob exists only to let us retrieve the configuration blob so that the manifest package can dig +// information out of it for Inspect(). +func (s *storageImageDestination) getConfigBlob(info types.BlobInfo) ([]byte, error) { + if info.Digest == "" { + return nil, errors.New(`no digest supplied when reading blob`) + } + if err := info.Digest.Validate(); err != nil { + return nil, perrors.Wrapf(err, `invalid digest supplied when reading blob`) + } + // Assume it's a file, since we're only calling this from a place that expects to read files. + if filename, ok := s.filenames[info.Digest]; ok { + contents, err2 := os.ReadFile(filename) + if err2 != nil { + return nil, perrors.Wrapf(err2, `reading blob from file %q`, filename) + } + return contents, nil + } + // If it's not a file, it's a bug, because we're not expecting to be asked for a layer. + return nil, errors.New("blob not found") +} + +// queueOrCommit queues in the specified blob to be committed to the storage. +// If no other goroutine is already committing layers, the layer and all +// subsequent layers (if already queued) will be committed to the storage. +func (s *storageImageDestination) queueOrCommit(ctx context.Context, blob types.BlobInfo, index int, emptyLayer bool) error { + // NOTE: whenever the code below is touched, make sure that all code + // paths unlock the lock and to unlock it exactly once. + // + // Conceptually, the code is divided in two stages: + // + // 1) Queue in work by marking the layer as ready to be committed. + // If at least one previous/parent layer with a lower index has + // not yet been committed, return early. + // + // 2) Process the queued-in work by committing the "ready" layers + // in sequence. Make sure that more items can be queued-in + // during the comparatively I/O expensive task of committing a + // layer. + // + // The conceptual benefit of this design is that caller can continue + // pulling layers after an early return. At any given time, only one + // caller is the "worker" routine committing layers. All other routines + // can continue pulling and queuing in layers. + s.lock.Lock() + s.indexToPulledLayerInfo[index] = &manifest.LayerInfo{ + BlobInfo: blob, + EmptyLayer: emptyLayer, + } + + // We're still waiting for at least one previous/parent layer to be + // committed, so there's nothing to do. + if index != s.currentIndex { + s.lock.Unlock() + return nil + } + + for info := s.indexToPulledLayerInfo[index]; info != nil; info = s.indexToPulledLayerInfo[index] { + s.lock.Unlock() + // Note: commitLayer locks on-demand. + if err := s.commitLayer(ctx, *info, index); err != nil { + return err + } + s.lock.Lock() + index++ + } + + // Set the index at the very end to make sure that only one routine + // enters stage 2). + s.currentIndex = index + s.lock.Unlock() + return nil +} + +// commitLayer commits the specified blob with the given index to the storage. +// Note that the previous layer is expected to already be committed. +// +// Caution: this function must be called without holding `s.lock`. Callers +// must guarantee that, at any given time, at most one goroutine may execute +// `commitLayer()`. +func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest.LayerInfo, index int) error { + // Already committed? Return early. + if _, alreadyCommitted := s.indexToStorageID[index]; alreadyCommitted { + return nil + } + + // Start with an empty string or the previous layer ID. Note that + // `s.indexToStorageID` can only be accessed by *one* goroutine at any + // given time. Hence, we don't need to lock accesses. + var lastLayer string + if prev := s.indexToStorageID[index-1]; prev != nil { + lastLayer = *prev + } + + // Carry over the previous ID for empty non-base layers. + if blob.EmptyLayer { + s.indexToStorageID[index] = &lastLayer + return nil + } + + // Check if there's already a layer with the ID that we'd give to the result of applying + // this layer blob to its parent, if it has one, or the blob's hex value otherwise. + s.lock.Lock() + diffID, haveDiffID := s.blobDiffIDs[blob.Digest] + s.lock.Unlock() + if !haveDiffID { + // Check if it's elsewhere and the caller just forgot to pass it to us in a PutBlob(), + // or to even check if we had it. + // Use none.NoCache to avoid a repeated DiffID lookup in the BlobInfoCache; a caller + // that relies on using a blob digest that has never been seen by the store had better call + // TryReusingBlob; not calling PutBlob already violates the documented API, so there’s only + // so far we are going to accommodate that (if we should be doing that at all). + logrus.Debugf("looking for diffID for blob %+v", blob.Digest) + // NOTE: use `TryReusingBlob` to prevent recursion. + has, _, err := s.TryReusingBlob(ctx, blob.BlobInfo, none.NoCache, false) + if err != nil { + return perrors.Wrapf(err, "checking for a layer based on blob %q", blob.Digest.String()) + } + if !has { + return fmt.Errorf("error determining uncompressed digest for blob %q", blob.Digest.String()) + } + diffID, haveDiffID = s.blobDiffIDs[blob.Digest] + if !haveDiffID { + return fmt.Errorf("we have blob %q, but don't know its uncompressed digest", blob.Digest.String()) + } + } + id := diffID.Hex() + if lastLayer != "" { + id = digest.Canonical.FromBytes([]byte(lastLayer + "+" + diffID.Hex())).Hex() + } + if layer, err2 := s.imageRef.transport.store.Layer(id); layer != nil && err2 == nil { + // There's already a layer that should have the right contents, just reuse it. + lastLayer = layer.ID + s.indexToStorageID[index] = &lastLayer + return nil + } + + s.lock.Lock() + diffOutput, ok := s.diffOutputs[blob.Digest] + s.lock.Unlock() + if ok { + layer, err := s.imageRef.transport.store.CreateLayer(id, lastLayer, nil, "", false, nil) + if err != nil { + return err + } + + // FIXME: what to do with the uncompressed digest? + diffOutput.UncompressedDigest = blob.Digest + + if err := s.imageRef.transport.store.ApplyDiffFromStagingDirectory(layer.ID, diffOutput.Target, diffOutput, nil); err != nil { + _ = s.imageRef.transport.store.Delete(layer.ID) + return err + } + + s.indexToStorageID[index] = &layer.ID + return nil + } + + s.lock.Lock() + al, ok := s.blobAdditionalLayer[blob.Digest] + s.lock.Unlock() + if ok { + layer, err := al.PutAs(id, lastLayer, nil) + if err != nil && !errors.Is(err, storage.ErrDuplicateID) { + return perrors.Wrapf(err, "failed to put layer from digest and labels") + } + lastLayer = layer.ID + s.indexToStorageID[index] = &lastLayer + return nil + } + + // Check if we previously cached a file with that blob's contents. If we didn't, + // then we need to read the desired contents from a layer. + s.lock.Lock() + filename, ok := s.filenames[blob.Digest] + s.lock.Unlock() + if !ok { + // Try to find the layer with contents matching that blobsum. + layer := "" + layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(diffID) + if err2 == nil && len(layers) > 0 { + layer = layers[0].ID + } else { + layers, err2 = s.imageRef.transport.store.LayersByCompressedDigest(blob.Digest) + if err2 == nil && len(layers) > 0 { + layer = layers[0].ID + } + } + if layer == "" { + return perrors.Wrapf(err2, "locating layer for blob %q", blob.Digest) + } + // Read the layer's contents. + noCompression := archive.Uncompressed + diffOptions := &storage.DiffOptions{ + Compression: &noCompression, + } + diff, err2 := s.imageRef.transport.store.Diff("", layer, diffOptions) + if err2 != nil { + return perrors.Wrapf(err2, "reading layer %q for blob %q", layer, blob.Digest) + } + // Copy the layer diff to a file. Diff() takes a lock that it holds + // until the ReadCloser that it returns is closed, and PutLayer() wants + // the same lock, so the diff can't just be directly streamed from one + // to the other. + filename = s.computeNextBlobCacheFile() + file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600) + if err != nil { + diff.Close() + return perrors.Wrapf(err, "creating temporary file %q", filename) + } + // Copy the data to the file. + // TODO: This can take quite some time, and should ideally be cancellable using + // ctx.Done(). + _, err = io.Copy(file, diff) + diff.Close() + file.Close() + if err != nil { + return perrors.Wrapf(err, "storing blob to file %q", filename) + } + // Make sure that we can find this file later, should we need the layer's + // contents again. + s.lock.Lock() + s.filenames[blob.Digest] = filename + s.lock.Unlock() + } + // Read the cached blob and use it as a diff. + file, err := os.Open(filename) + if err != nil { + return perrors.Wrapf(err, "opening file %q", filename) + } + defer file.Close() + // Build the new layer using the diff, regardless of where it came from. + // TODO: This can take quite some time, and should ideally be cancellable using ctx.Done(). + layer, _, err := s.imageRef.transport.store.PutLayer(id, lastLayer, nil, "", false, &storage.LayerOptions{ + OriginalDigest: blob.Digest, + UncompressedDigest: diffID, + }, file) + if err != nil && !errors.Is(err, storage.ErrDuplicateID) { + return perrors.Wrapf(err, "adding layer with blob %q", blob.Digest) + } + + s.indexToStorageID[index] = &layer.ID + return nil +} + +// Commit marks the process of storing the image as successful and asks for the image to be persisted. +// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list +// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the +// original manifest list digest, if desired. +// WARNING: This does not have any transactional semantics: +// - Uploaded data MAY be visible to others before Commit() is called +// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) +func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error { + if len(s.manifest) == 0 { + return errors.New("Internal error: storageImageDestination.Commit() called without PutManifest()") + } + toplevelManifest, _, err := unparsedToplevel.Manifest(ctx) + if err != nil { + return perrors.Wrapf(err, "retrieving top-level manifest") + } + // If the name we're saving to includes a digest, then check that the + // manifests that we're about to save all either match the one from the + // unparsedToplevel, or match the digest in the name that we're using. + if s.imageRef.named != nil { + if digested, ok := s.imageRef.named.(reference.Digested); ok { + matches, err := manifest.MatchesDigest(s.manifest, digested.Digest()) + if err != nil { + return err + } + if !matches { + matches, err = manifest.MatchesDigest(toplevelManifest, digested.Digest()) + if err != nil { + return err + } + } + if !matches { + return fmt.Errorf("Manifest to be saved does not match expected digest %s", digested.Digest()) + } + } + } + // Find the list of layer blobs. + man, err := manifest.FromBlob(s.manifest, manifest.GuessMIMEType(s.manifest)) + if err != nil { + return perrors.Wrapf(err, "parsing manifest") + } + layerBlobs := man.LayerInfos() + + // Extract, commit, or find the layers. + for i, blob := range layerBlobs { + if err := s.commitLayer(ctx, blob, i); err != nil { + return err + } + } + var lastLayer string + if len(layerBlobs) > 0 { // Can happen when using caches + prev := s.indexToStorageID[len(layerBlobs)-1] + if prev == nil { + return fmt.Errorf("Internal error: StorageImageDestination.Commit(): previous layer %d hasn't been committed (lastLayer == nil)", len(layerBlobs)-1) + } + lastLayer = *prev + } + + // If one of those blobs was a configuration blob, then we can try to dig out the date when the image + // was originally created, in case we're just copying it. If not, no harm done. + options := &storage.ImageOptions{} + if inspect, err := man.Inspect(s.getConfigBlob); err == nil && inspect.Created != nil { + logrus.Debugf("setting image creation date to %s", inspect.Created) + options.CreationDate = *inspect.Created + } + // Create the image record, pointing to the most-recently added layer. + intendedID := s.imageRef.id + if intendedID == "" { + intendedID = s.computeID(man) + } + oldNames := []string{} + img, err := s.imageRef.transport.store.CreateImage(intendedID, nil, lastLayer, "", options) + if err != nil { + if !errors.Is(err, storage.ErrDuplicateID) { + logrus.Debugf("error creating image: %q", err) + return perrors.Wrapf(err, "creating image %q", intendedID) + } + img, err = s.imageRef.transport.store.Image(intendedID) + if err != nil { + return perrors.Wrapf(err, "reading image %q", intendedID) + } + if img.TopLayer != lastLayer { + logrus.Debugf("error creating image: image with ID %q exists, but uses different layers", intendedID) + return perrors.Wrapf(storage.ErrDuplicateID, "image with ID %q already exists, but uses a different top layer", intendedID) + } + logrus.Debugf("reusing image ID %q", img.ID) + oldNames = append(oldNames, img.Names...) + } else { + logrus.Debugf("created new image ID %q", img.ID) + } + + // Clean up the unfinished image on any error. + // (Is this the right thing to do if the image has existed before?) + commitSucceeded := false + defer func() { + if !commitSucceeded { + logrus.Errorf("Updating image %q (old names %v) failed, deleting it", img.ID, oldNames) + if _, err := s.imageRef.transport.store.DeleteImage(img.ID, true); err != nil { + logrus.Errorf("Error deleting incomplete image %q: %v", img.ID, err) + } + } + }() + + // Add the non-layer blobs as data items. Since we only share layers, they should all be in files, so + // we just need to screen out the ones that are actually layers to get the list of non-layers. + dataBlobs := make(map[digest.Digest]struct{}) + for blob := range s.filenames { + dataBlobs[blob] = struct{}{} + } + for _, layerBlob := range layerBlobs { + delete(dataBlobs, layerBlob.Digest) + } + for blob := range dataBlobs { + v, err := os.ReadFile(s.filenames[blob]) + if err != nil { + return perrors.Wrapf(err, "copying non-layer blob %q to image", blob) + } + if err := s.imageRef.transport.store.SetImageBigData(img.ID, blob.String(), v, manifest.Digest); err != nil { + logrus.Debugf("error saving big data %q for image %q: %v", blob.String(), img.ID, err) + return perrors.Wrapf(err, "saving big data %q for image %q", blob.String(), img.ID) + } + } + // Save the unparsedToplevel's manifest if it differs from the per-platform one, which is saved below. + if len(toplevelManifest) != 0 && !bytes.Equal(toplevelManifest, s.manifest) { + manifestDigest, err := manifest.Digest(toplevelManifest) + if err != nil { + return perrors.Wrapf(err, "digesting top-level manifest") + } + key := manifestBigDataKey(manifestDigest) + if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, toplevelManifest, manifest.Digest); err != nil { + logrus.Debugf("error saving top-level manifest for image %q: %v", img.ID, err) + return perrors.Wrapf(err, "saving top-level manifest for image %q", img.ID) + } + } + // Save the image's manifest. Allow looking it up by digest by using the key convention defined by the Store. + // Record the manifest twice: using a digest-specific key to allow references to that specific digest instance, + // and using storage.ImageDigestBigDataKey for future users that don’t specify any digest and for compatibility with older readers. + key := manifestBigDataKey(s.manifestDigest) + if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil { + logrus.Debugf("error saving manifest for image %q: %v", img.ID, err) + return perrors.Wrapf(err, "saving manifest for image %q", img.ID) + } + key = storage.ImageDigestBigDataKey + if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil { + logrus.Debugf("error saving manifest for image %q: %v", img.ID, err) + return perrors.Wrapf(err, "saving manifest for image %q", img.ID) + } + // Save the signatures, if we have any. + if len(s.signatures) > 0 { + if err := s.imageRef.transport.store.SetImageBigData(img.ID, "signatures", s.signatures, manifest.Digest); err != nil { + logrus.Debugf("error saving signatures for image %q: %v", img.ID, err) + return perrors.Wrapf(err, "saving signatures for image %q", img.ID) + } + } + for instanceDigest, signatures := range s.signatureses { + key := signatureBigDataKey(instanceDigest) + if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, signatures, manifest.Digest); err != nil { + logrus.Debugf("error saving signatures for image %q: %v", img.ID, err) + return perrors.Wrapf(err, "saving signatures for image %q", img.ID) + } + } + // Save our metadata. + metadata, err := json.Marshal(s) + if err != nil { + logrus.Debugf("error encoding metadata for image %q: %v", img.ID, err) + return perrors.Wrapf(err, "encoding metadata for image %q", img.ID) + } + if len(metadata) != 0 { + if err = s.imageRef.transport.store.SetMetadata(img.ID, string(metadata)); err != nil { + logrus.Debugf("error saving metadata for image %q: %v", img.ID, err) + return perrors.Wrapf(err, "saving metadata for image %q", img.ID) + } + logrus.Debugf("saved image metadata %q", string(metadata)) + } + // Adds the reference's name on the image. We don't need to worry about avoiding duplicate + // values because AddNames() will deduplicate the list that we pass to it. + if name := s.imageRef.DockerReference(); name != nil { + if err := s.imageRef.transport.store.AddNames(img.ID, []string{name.String()}); err != nil { + return perrors.Wrapf(err, "adding names %v to image %q", name, img.ID) + } + logrus.Debugf("added name %q to image %q", name, img.ID) + } + + commitSucceeded = true + return nil +} + +// PutManifest writes the manifest to the destination. +func (s *storageImageDestination) PutManifest(ctx context.Context, manifestBlob []byte, instanceDigest *digest.Digest) error { + digest, err := manifest.Digest(manifestBlob) + if err != nil { + return err + } + newBlob := make([]byte, len(manifestBlob)) + copy(newBlob, manifestBlob) + s.manifest = newBlob + s.manifestDigest = digest + return nil +} + +// PutSignatures records the image's signatures for committing as a single data blob. +func (s *storageImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { + sizes := []int{} + sigblob := []byte{} + for _, sig := range signatures { + sizes = append(sizes, len(sig)) + newblob := make([]byte, len(sigblob)+len(sig)) + copy(newblob, sigblob) + copy(newblob[len(sigblob):], sig) + sigblob = newblob + } + if instanceDigest == nil { + s.signatures = sigblob + s.SignatureSizes = sizes + if len(s.manifest) > 0 { + manifestDigest := s.manifestDigest + instanceDigest = &manifestDigest + } + } + if instanceDigest != nil { + s.signatureses[*instanceDigest] = sigblob + s.SignaturesSizes[*instanceDigest] = sizes + } + return nil +} diff --git a/storage/storage_image.go b/storage/storage_image.go index e81f0ea9c..9f16dd334 100644 --- a/storage/storage_image.go +++ b/storage/storage_image.go @@ -4,101 +4,20 @@ package storage import ( - "bytes" "context" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "sync" - "sync/atomic" - "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/internal/blobinfocache" "github.com/containers/image/v5/internal/image" - "github.com/containers/image/v5/internal/imagedestination/impl" - "github.com/containers/image/v5/internal/imagedestination/stubs" - "github.com/containers/image/v5/internal/private" - "github.com/containers/image/v5/internal/putblobdigest" - "github.com/containers/image/v5/internal/tmpdir" - "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/pkg/blobinfocache/none" "github.com/containers/image/v5/types" "github.com/containers/storage" - graphdriver "github.com/containers/storage/drivers" - "github.com/containers/storage/pkg/archive" - "github.com/containers/storage/pkg/chunked" - "github.com/containers/storage/pkg/ioutils" digest "github.com/opencontainers/go-digest" - imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" - perrors "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) var ( - // ErrBlobDigestMismatch could potentially be returned when PutBlob() is given a blob - // with a digest-based name that doesn't match its contents. - // Deprecated: PutBlob() doesn't do this any more (it just accepts the caller’s value), - // and there is no known user of this error. - ErrBlobDigestMismatch = errors.New("blob digest mismatch") - // ErrBlobSizeMismatch is returned when PutBlob() is given a blob - // with an expected size that doesn't match the reader. - ErrBlobSizeMismatch = errors.New("blob size mismatch") // ErrNoSuchImage is returned when we attempt to access an image which // doesn't exist in the storage area. ErrNoSuchImage = storage.ErrNotAnImage ) -type storageImageSource struct { - imageRef storageReference - image *storage.Image - systemContext *types.SystemContext // SystemContext used in GetBlob() to create temporary files - layerPosition map[digest.Digest]int // Where we are in reading a blob's layers - cachedManifest []byte // A cached copy of the manifest, if already known, or nil - getBlobMutex sync.Mutex // Mutex to sync state for parallel GetBlob executions - SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice - SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // List of sizes of each signature slice -} - -type storageImageDestination struct { - impl.Compat - impl.PropertyMethodsInitialize - stubs.ImplementsPutBlobPartial - stubs.AlwaysSupportsSignatures - - imageRef storageReference - directory string // Temporary directory where we store blobs until Commit() time - nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs - manifest []byte // Manifest contents, temporary - manifestDigest digest.Digest // Valid if len(manifest) != 0 - signatures []byte // Signature contents, temporary - signatureses map[digest.Digest][]byte // Instance signature contents, temporary - SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice - SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // Sizes of each manifest's signature slice - - // A storage destination may be used concurrently. Accesses are - // serialized via a mutex. Please refer to the individual comments - // below for details. - lock sync.Mutex - // Mapping from layer (by index) to the associated ID in the storage. - // It's protected *implicitly* since `commitLayer()`, at any given - // time, can only be executed by *one* goroutine. Please refer to - // `queueOrCommit()` for further details on how the single-caller - // guarantee is implemented. - indexToStorageID map[int]*string - // All accesses to below data are protected by `lock` which is made - // *explicit* in the code. - blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs - fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes - filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them - currentIndex int // The index of the layer to be committed (i.e., lower indices have already been committed) - indexToPulledLayerInfo map[int]*manifest.LayerInfo // Mapping from layer (by index) to pulled down blob - blobAdditionalLayer map[digest.Digest]storage.AdditionalLayer // Mapping from layer blobsums to their corresponding additional layer - diffOutputs map[digest.Digest]*graphdriver.DriverWithDifferOutput // Mapping from digest to differ output -} - type storageImageCloser struct { types.ImageCloser size int64 @@ -117,1165 +36,6 @@ func signatureBigDataKey(digest digest.Digest) string { return "signature-" + digest.Encoded() } -// newImageSource sets up an image for reading. -func newImageSource(ctx context.Context, sys *types.SystemContext, imageRef storageReference) (*storageImageSource, error) { - // First, locate the image. - img, err := imageRef.resolveImage(sys) - if err != nil { - return nil, err - } - - // Build the reader object. - image := &storageImageSource{ - imageRef: imageRef, - systemContext: sys, - image: img, - layerPosition: make(map[digest.Digest]int), - SignatureSizes: []int{}, - SignaturesSizes: make(map[digest.Digest][]int), - } - if img.Metadata != "" { - if err := json.Unmarshal([]byte(img.Metadata), image); err != nil { - return nil, perrors.Wrap(err, "decoding metadata for source image") - } - } - return image, nil -} - -// Reference returns the image reference that we used to find this image. -func (s *storageImageSource) Reference() types.ImageReference { - return s.imageRef -} - -// Close cleans up any resources we tied up while reading the image. -func (s *storageImageSource) Close() error { - return nil -} - -// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. -func (s *storageImageSource) HasThreadSafeGetBlob() bool { - return true -} - -// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). -// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. -// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. -func (s *storageImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (rc io.ReadCloser, n int64, err error) { - if info.Digest == image.GzippedEmptyLayerDigest { - return io.NopCloser(bytes.NewReader(image.GzippedEmptyLayer)), int64(len(image.GzippedEmptyLayer)), nil - } - - // NOTE: the blob is first written to a temporary file and subsequently - // closed. The intention is to keep the time we own the storage lock - // as short as possible to allow other processes to access the storage. - rc, n, _, err = s.getBlobAndLayerID(info) - if err != nil { - return nil, 0, err - } - defer rc.Close() - - tmpFile, err := os.CreateTemp(tmpdir.TemporaryDirectoryForBigFiles(s.systemContext), "") - if err != nil { - return nil, 0, err - } - - if _, err := io.Copy(tmpFile, rc); err != nil { - return nil, 0, err - } - - if _, err := tmpFile.Seek(0, 0); err != nil { - return nil, 0, err - } - - wrapper := ioutils.NewReadCloserWrapper(tmpFile, func() error { - defer os.Remove(tmpFile.Name()) - return tmpFile.Close() - }) - - return wrapper, n, err -} - -// getBlobAndLayer reads the data blob or filesystem layer which matches the digest and size, if given. -func (s *storageImageSource) getBlobAndLayerID(info types.BlobInfo) (rc io.ReadCloser, n int64, layerID string, err error) { - var layer storage.Layer - var diffOptions *storage.DiffOptions - // We need a valid digest value. - err = info.Digest.Validate() - if err != nil { - return nil, -1, "", err - } - // Check if the blob corresponds to a diff that was used to initialize any layers. Our - // callers should try to retrieve layers using their uncompressed digests, so no need to - // check if they're using one of the compressed digests, which we can't reproduce anyway. - layers, _ := s.imageRef.transport.store.LayersByUncompressedDigest(info.Digest) - - // If it's not a layer, then it must be a data item. - if len(layers) == 0 { - b, err := s.imageRef.transport.store.ImageBigData(s.image.ID, info.Digest.String()) - if err != nil { - return nil, -1, "", err - } - r := bytes.NewReader(b) - logrus.Debugf("exporting opaque data as blob %q", info.Digest.String()) - return io.NopCloser(r), int64(r.Len()), "", nil - } - // Step through the list of matching layers. Tests may want to verify that if we have multiple layers - // which claim to have the same contents, that we actually do have multiple layers, otherwise we could - // just go ahead and use the first one every time. - s.getBlobMutex.Lock() - i := s.layerPosition[info.Digest] - s.layerPosition[info.Digest] = i + 1 - s.getBlobMutex.Unlock() - if len(layers) > 0 { - layer = layers[i%len(layers)] - } - // Force the storage layer to not try to match any compression that was used when the layer was first - // handed to it. - noCompression := archive.Uncompressed - diffOptions = &storage.DiffOptions{ - Compression: &noCompression, - } - if layer.UncompressedSize < 0 { - n = -1 - } else { - n = layer.UncompressedSize - } - logrus.Debugf("exporting filesystem layer %q without compression for blob %q", layer.ID, info.Digest) - rc, err = s.imageRef.transport.store.Diff("", layer.ID, diffOptions) - if err != nil { - return nil, -1, "", err - } - return rc, n, layer.ID, err -} - -// GetManifest() reads the image's manifest. -func (s *storageImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) (manifestBlob []byte, MIMEType string, err error) { - if instanceDigest != nil { - key := manifestBigDataKey(*instanceDigest) - blob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key) - if err != nil { - return nil, "", perrors.Wrapf(err, "reading manifest for image instance %q", *instanceDigest) - } - return blob, manifest.GuessMIMEType(blob), err - } - if len(s.cachedManifest) == 0 { - // The manifest is stored as a big data item. - // Prefer the manifest corresponding to the user-specified digest, if available. - if s.imageRef.named != nil { - if digested, ok := s.imageRef.named.(reference.Digested); ok { - key := manifestBigDataKey(digested.Digest()) - blob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key) - if err != nil && !os.IsNotExist(err) { // os.IsNotExist is true if the image exists but there is no data corresponding to key - return nil, "", err - } - if err == nil { - s.cachedManifest = blob - } - } - } - // If the user did not specify a digest, or this is an old image stored before manifestBigDataKey was introduced, use the default manifest. - // Note that the manifest may not match the expected digest, and that is likely to fail eventually, e.g. in c/image/image/UnparsedImage.Manifest(). - if len(s.cachedManifest) == 0 { - cachedBlob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, storage.ImageDigestBigDataKey) - if err != nil { - return nil, "", err - } - s.cachedManifest = cachedBlob - } - } - return s.cachedManifest, manifest.GuessMIMEType(s.cachedManifest), err -} - -// LayerInfosForCopy() returns the list of layer blobs that make up the root filesystem of -// the image, after they've been decompressed. -func (s *storageImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { - manifestBlob, manifestType, err := s.GetManifest(ctx, instanceDigest) - if err != nil { - return nil, perrors.Wrapf(err, "reading image manifest for %q", s.image.ID) - } - if manifest.MIMETypeIsMultiImage(manifestType) { - return nil, errors.New("can't copy layers for a manifest list (shouldn't be attempted)") - } - man, err := manifest.FromBlob(manifestBlob, manifestType) - if err != nil { - return nil, perrors.Wrapf(err, "parsing image manifest for %q", s.image.ID) - } - - uncompressedLayerType := "" - switch manifestType { - case imgspecv1.MediaTypeImageManifest: - uncompressedLayerType = imgspecv1.MediaTypeImageLayer - case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema2MediaType: - uncompressedLayerType = manifest.DockerV2SchemaLayerMediaTypeUncompressed - } - - physicalBlobInfos := []types.BlobInfo{} - layerID := s.image.TopLayer - for layerID != "" { - layer, err := s.imageRef.transport.store.Layer(layerID) - if err != nil { - return nil, perrors.Wrapf(err, "reading layer %q in image %q", layerID, s.image.ID) - } - if layer.UncompressedDigest == "" { - return nil, fmt.Errorf("uncompressed digest for layer %q is unknown", layerID) - } - if layer.UncompressedSize < 0 { - return nil, fmt.Errorf("uncompressed size for layer %q is unknown", layerID) - } - blobInfo := types.BlobInfo{ - Digest: layer.UncompressedDigest, - Size: layer.UncompressedSize, - MediaType: uncompressedLayerType, - } - physicalBlobInfos = append([]types.BlobInfo{blobInfo}, physicalBlobInfos...) - layerID = layer.Parent - } - - res, err := buildLayerInfosForCopy(man.LayerInfos(), physicalBlobInfos) - if err != nil { - return nil, perrors.Wrapf(err, "creating LayerInfosForCopy of image %q", s.image.ID) - } - return res, nil -} - -// buildLayerInfosForCopy builds a LayerInfosForCopy return value based on manifestInfos from the original manifest, -// but using layer data which we can actually produce — physicalInfos for non-empty layers, -// and image.GzippedEmptyLayer for empty ones. -// (This is split basically only to allow easily unit-testing the part that has no dependencies on the external environment.) -func buildLayerInfosForCopy(manifestInfos []manifest.LayerInfo, physicalInfos []types.BlobInfo) ([]types.BlobInfo, error) { - nextPhysical := 0 - res := make([]types.BlobInfo, len(manifestInfos)) - for i, mi := range manifestInfos { - if mi.EmptyLayer { - res[i] = types.BlobInfo{ - Digest: image.GzippedEmptyLayerDigest, - Size: int64(len(image.GzippedEmptyLayer)), - MediaType: mi.MediaType, - } - } else { - if nextPhysical >= len(physicalInfos) { - return nil, fmt.Errorf("expected more than %d physical layers to exist", len(physicalInfos)) - } - res[i] = physicalInfos[nextPhysical] - nextPhysical++ - } - } - if nextPhysical != len(physicalInfos) { - return nil, fmt.Errorf("used only %d out of %d physical layers", nextPhysical, len(physicalInfos)) - } - return res, nil -} - -// GetSignatures() parses the image's signatures blob into a slice of byte slices. -func (s *storageImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) (signatures [][]byte, err error) { - var offset int - sigslice := [][]byte{} - signature := []byte{} - signatureSizes := s.SignatureSizes - key := "signatures" - instance := "default instance" - if instanceDigest != nil { - signatureSizes = s.SignaturesSizes[*instanceDigest] - key = signatureBigDataKey(*instanceDigest) - instance = instanceDigest.Encoded() - } - if len(signatureSizes) > 0 { - signatureBlob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key) - if err != nil { - return nil, perrors.Wrapf(err, "looking up signatures data for image %q (%s)", s.image.ID, instance) - } - signature = signatureBlob - } - for _, length := range signatureSizes { - if offset+length > len(signature) { - return nil, perrors.Wrapf(err, "looking up signatures data for image %q (%s): expected at least %d bytes, only found %d", s.image.ID, instance, len(signature), offset+length) - } - sigslice = append(sigslice, signature[offset:offset+length]) - offset += length - } - if offset != len(signature) { - return nil, fmt.Errorf("signatures data (%s) contained %d extra bytes", instance, len(signatures)-offset) - } - return sigslice, nil -} - -// newImageDestination sets us up to write a new image, caching blobs in a temporary directory until -// it's time to Commit() the image -func newImageDestination(sys *types.SystemContext, imageRef storageReference) (*storageImageDestination, error) { - directory, err := os.MkdirTemp(tmpdir.TemporaryDirectoryForBigFiles(sys), "storage") - if err != nil { - return nil, perrors.Wrapf(err, "creating a temporary directory") - } - dest := &storageImageDestination{ - PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ - SupportedManifestMIMETypes: []string{ - imgspecv1.MediaTypeImageManifest, - manifest.DockerV2Schema2MediaType, - manifest.DockerV2Schema1SignedMediaType, - manifest.DockerV2Schema1MediaType, - }, - // We ultimately have to decompress layers to populate trees on disk - // and need to explicitly ask for it here, so that the layers' MIME - // types can be set accordingly. - DesiredLayerCompression: types.PreserveOriginal, - AcceptsForeignLayerURLs: false, - MustMatchRuntimeOS: true, - IgnoresEmbeddedDockerReference: true, // Yes, we want the unmodified manifest - HasThreadSafePutBlob: true, - }), - - imageRef: imageRef, - directory: directory, - signatureses: make(map[digest.Digest][]byte), - blobDiffIDs: make(map[digest.Digest]digest.Digest), - blobAdditionalLayer: make(map[digest.Digest]storage.AdditionalLayer), - fileSizes: make(map[digest.Digest]int64), - filenames: make(map[digest.Digest]string), - SignatureSizes: []int{}, - SignaturesSizes: make(map[digest.Digest][]int), - indexToStorageID: make(map[int]*string), - indexToPulledLayerInfo: make(map[int]*manifest.LayerInfo), - diffOutputs: make(map[digest.Digest]*graphdriver.DriverWithDifferOutput), - } - dest.Compat = impl.AddCompat(dest) - return dest, nil -} - -// Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, -// e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects. -func (s *storageImageDestination) Reference() types.ImageReference { - return s.imageRef -} - -// Close cleans up the temporary directory and additional layer store handlers. -func (s *storageImageDestination) Close() error { - for _, al := range s.blobAdditionalLayer { - al.Release() - } - for _, v := range s.diffOutputs { - if v.Target != "" { - _ = s.imageRef.transport.store.CleanupStagingDirectory(v.Target) - } - } - return os.RemoveAll(s.directory) -} - -func (s *storageImageDestination) computeNextBlobCacheFile() string { - return filepath.Join(s.directory, fmt.Sprintf("%d", atomic.AddInt32(&s.nextTempFileID, 1))) -} - -// PutBlobWithOptions writes contents of stream and returns data representing the result. -// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. -// inputInfo.Size is the expected length of stream, if known. -// inputInfo.MediaType describes the blob format, if known. -// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available -// to any other readers for download using the supplied digest. -// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (s *storageImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, options private.PutBlobOptions) (types.BlobInfo, error) { - info, err := s.putBlobToPendingFile(ctx, stream, blobinfo, &options) - if err != nil { - return info, err - } - - if options.IsConfig || options.LayerIndex == nil { - return info, nil - } - - return info, s.queueOrCommit(ctx, info, *options.LayerIndex, options.EmptyLayer) -} - -// putBlobToPendingFile implements ImageDestination.PutBlobWithOptions, storing stream into an on-disk file. -// The caller must arrange the blob to be eventually committed using s.commitLayer(). -func (s *storageImageDestination) putBlobToPendingFile(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, options *private.PutBlobOptions) (types.BlobInfo, error) { - // Stores a layer or data blob in our temporary directory, checking that any information - // in the blobinfo matches the incoming data. - errorBlobInfo := types.BlobInfo{ - Digest: "", - Size: -1, - } - if blobinfo.Digest != "" { - if err := blobinfo.Digest.Validate(); err != nil { - return errorBlobInfo, fmt.Errorf("invalid digest %#v: %w", blobinfo.Digest.String(), err) - } - } - - // Set up to digest the blob if necessary, and count its size while saving it to a file. - filename := s.computeNextBlobCacheFile() - file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600) - if err != nil { - return errorBlobInfo, perrors.Wrapf(err, "creating temporary file %q", filename) - } - defer file.Close() - counter := ioutils.NewWriteCounter(file) - stream = io.TeeReader(stream, counter) - digester, stream := putblobdigest.DigestIfUnknown(stream, blobinfo) - decompressed, err := archive.DecompressStream(stream) - if err != nil { - return errorBlobInfo, perrors.Wrap(err, "setting up to decompress blob") - } - - diffID := digest.Canonical.Digester() - // Copy the data to the file. - // TODO: This can take quite some time, and should ideally be cancellable using ctx.Done(). - _, err = io.Copy(diffID.Hash(), decompressed) - decompressed.Close() - if err != nil { - return errorBlobInfo, perrors.Wrapf(err, "storing blob to file %q", filename) - } - - // Determine blob properties, and fail if information that we were given about the blob - // is known to be incorrect. - blobDigest := digester.Digest() - blobSize := blobinfo.Size - if blobSize < 0 { - blobSize = counter.Count - } else if blobinfo.Size != counter.Count { - return errorBlobInfo, ErrBlobSizeMismatch - } - - // Record information about the blob. - s.lock.Lock() - s.blobDiffIDs[blobDigest] = diffID.Digest() - s.fileSizes[blobDigest] = counter.Count - s.filenames[blobDigest] = filename - s.lock.Unlock() - // This is safe because we have just computed diffID, and blobDigest was either computed - // by us, or validated by the caller (usually copy.digestingReader). - options.Cache.RecordDigestUncompressedPair(blobDigest, diffID.Digest()) - return types.BlobInfo{ - Digest: blobDigest, - Size: blobSize, - MediaType: blobinfo.MediaType, - }, nil -} - -type zstdFetcher struct { - chunkAccessor private.BlobChunkAccessor - ctx context.Context - blobInfo types.BlobInfo -} - -// GetBlobAt converts from chunked.GetBlobAt to BlobChunkAccessor.GetBlobAt. -func (f *zstdFetcher) GetBlobAt(chunks []chunked.ImageSourceChunk) (chan io.ReadCloser, chan error, error) { - var newChunks []private.ImageSourceChunk - for _, v := range chunks { - i := private.ImageSourceChunk{ - Offset: v.Offset, - Length: v.Length, - } - newChunks = append(newChunks, i) - } - rc, errs, err := f.chunkAccessor.GetBlobAt(f.ctx, f.blobInfo, newChunks) - if _, ok := err.(private.BadPartialRequestError); ok { - err = chunked.ErrBadRequest{} - } - return rc, errs, err - -} - -// PutBlobPartial attempts to create a blob using the data that is already present -// at the destination. chunkAccessor is accessed in a non-sequential way to retrieve the missing chunks. -// It is available only if SupportsPutBlobPartial(). -// Even if SupportsPutBlobPartial() returns true, the call can fail, in which case the caller -// should fall back to PutBlobWithOptions. -func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, cache blobinfocache.BlobInfoCache2) (types.BlobInfo, error) { - fetcher := zstdFetcher{ - chunkAccessor: chunkAccessor, - ctx: ctx, - blobInfo: srcInfo, - } - - differ, err := chunked.GetDiffer(ctx, s.imageRef.transport.store, srcInfo.Size, srcInfo.Annotations, &fetcher) - if err != nil { - return srcInfo, err - } - - out, err := s.imageRef.transport.store.ApplyDiffWithDiffer("", nil, differ) - if err != nil { - return srcInfo, err - } - - blobDigest := srcInfo.Digest - - s.lock.Lock() - s.blobDiffIDs[blobDigest] = blobDigest - s.fileSizes[blobDigest] = 0 - s.filenames[blobDigest] = "" - s.diffOutputs[blobDigest] = out - s.lock.Unlock() - - return srcInfo, nil -} - -// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination -// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). -// info.Digest must not be empty. -// If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may -// include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be -// reflected in the manifest that will be written. -// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. -func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context, blobinfo types.BlobInfo, options private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { - reused, info, err := s.tryReusingBlobAsPending(ctx, blobinfo, &options) - if err != nil || !reused || options.LayerIndex == nil { - return reused, info, err - } - - return reused, info, s.queueOrCommit(ctx, info, *options.LayerIndex, options.EmptyLayer) -} - -// tryReusingBlobAsPending implements TryReusingBlobWithOptions, filling s.blobDiffIDs and other metadata. -// The caller must arrange the blob to be eventually committed using s.commitLayer(). -func (s *storageImageDestination) tryReusingBlobAsPending(ctx context.Context, blobinfo types.BlobInfo, options *private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { - // lock the entire method as it executes fairly quickly - s.lock.Lock() - defer s.lock.Unlock() - - if options.SrcRef != nil { - // Check if we have the layer in the underlying additional layer store. - aLayer, err := s.imageRef.transport.store.LookupAdditionalLayer(blobinfo.Digest, options.SrcRef.String()) - if err != nil && !errors.Is(err, storage.ErrLayerUnknown) { - return false, types.BlobInfo{}, perrors.Wrapf(err, `looking for compressed layers with digest %q and labels`, blobinfo.Digest) - } else if err == nil { - // Record the uncompressed value so that we can use it to calculate layer IDs. - s.blobDiffIDs[blobinfo.Digest] = aLayer.UncompressedDigest() - s.blobAdditionalLayer[blobinfo.Digest] = aLayer - return true, types.BlobInfo{ - Digest: blobinfo.Digest, - Size: aLayer.CompressedSize(), - MediaType: blobinfo.MediaType, - }, nil - } - } - - if blobinfo.Digest == "" { - return false, types.BlobInfo{}, errors.New(`Can not check for a blob with unknown digest`) - } - if err := blobinfo.Digest.Validate(); err != nil { - return false, types.BlobInfo{}, perrors.Wrapf(err, `Can not check for a blob with invalid digest`) - } - - // Check if we've already cached it in a file. - if size, ok := s.fileSizes[blobinfo.Digest]; ok { - return true, types.BlobInfo{ - Digest: blobinfo.Digest, - Size: size, - MediaType: blobinfo.MediaType, - }, nil - } - - // Check if we have a wasn't-compressed layer in storage that's based on that blob. - layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(blobinfo.Digest) - if err != nil && !errors.Is(err, storage.ErrLayerUnknown) { - return false, types.BlobInfo{}, perrors.Wrapf(err, `looking for layers with digest %q`, blobinfo.Digest) - } - if len(layers) > 0 { - // Save this for completeness. - s.blobDiffIDs[blobinfo.Digest] = layers[0].UncompressedDigest - return true, types.BlobInfo{ - Digest: blobinfo.Digest, - Size: layers[0].UncompressedSize, - MediaType: blobinfo.MediaType, - }, nil - } - - // Check if we have a was-compressed layer in storage that's based on that blob. - layers, err = s.imageRef.transport.store.LayersByCompressedDigest(blobinfo.Digest) - if err != nil && !errors.Is(err, storage.ErrLayerUnknown) { - return false, types.BlobInfo{}, perrors.Wrapf(err, `looking for compressed layers with digest %q`, blobinfo.Digest) - } - if len(layers) > 0 { - // Record the uncompressed value so that we can use it to calculate layer IDs. - s.blobDiffIDs[blobinfo.Digest] = layers[0].UncompressedDigest - return true, types.BlobInfo{ - Digest: blobinfo.Digest, - Size: layers[0].CompressedSize, - MediaType: blobinfo.MediaType, - }, nil - } - - // Does the blob correspond to a known DiffID which we already have available? - // Because we must return the size, which is unknown for unavailable compressed blobs, the returned BlobInfo refers to the - // uncompressed layer, and that can happen only if options.CanSubstitute, or if the incoming manifest already specifies the size. - if options.CanSubstitute || blobinfo.Size != -1 { - if uncompressedDigest := options.Cache.UncompressedDigest(blobinfo.Digest); uncompressedDigest != "" && uncompressedDigest != blobinfo.Digest { - layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(uncompressedDigest) - if err != nil && !errors.Is(err, storage.ErrLayerUnknown) { - return false, types.BlobInfo{}, perrors.Wrapf(err, `looking for layers with digest %q`, uncompressedDigest) - } - if len(layers) > 0 { - if blobinfo.Size != -1 { - s.blobDiffIDs[blobinfo.Digest] = layers[0].UncompressedDigest - return true, blobinfo, nil - } - if !options.CanSubstitute { - return false, types.BlobInfo{}, fmt.Errorf("Internal error: options.CanSubstitute was expected to be true for blobInfo %v", blobinfo) - } - s.blobDiffIDs[uncompressedDigest] = layers[0].UncompressedDigest - return true, types.BlobInfo{ - Digest: uncompressedDigest, - Size: layers[0].UncompressedSize, - MediaType: blobinfo.MediaType, - }, nil - } - } - } - - // Nope, we don't have it. - return false, types.BlobInfo{}, nil -} - -// computeID computes a recommended image ID based on information we have so far. If -// the manifest is not of a type that we recognize, we return an empty value, indicating -// that since we don't have a recommendation, a random ID should be used if one needs -// to be allocated. -func (s *storageImageDestination) computeID(m manifest.Manifest) string { - // Build the diffID list. We need the decompressed sums that we've been calculating to - // fill in the DiffIDs. It's expected (but not enforced by us) that the number of - // diffIDs corresponds to the number of non-EmptyLayer entries in the history. - var diffIDs []digest.Digest - switch m := m.(type) { - case *manifest.Schema1: - // Build a list of the diffIDs we've generated for the non-throwaway FS layers, - // in reverse of the order in which they were originally listed. - for i, compat := range m.ExtractedV1Compatibility { - if compat.ThrowAway { - continue - } - blobSum := m.FSLayers[i].BlobSum - diffID, ok := s.blobDiffIDs[blobSum] - if !ok { - logrus.Infof("error looking up diffID for layer %q", blobSum.String()) - return "" - } - diffIDs = append([]digest.Digest{diffID}, diffIDs...) - } - case *manifest.Schema2, *manifest.OCI1: - // We know the ID calculation for these formats doesn't actually use the diffIDs, - // so we don't need to populate the diffID list. - default: - return "" - } - id, err := m.ImageID(diffIDs) - if err != nil { - return "" - } - return id -} - -// getConfigBlob exists only to let us retrieve the configuration blob so that the manifest package can dig -// information out of it for Inspect(). -func (s *storageImageDestination) getConfigBlob(info types.BlobInfo) ([]byte, error) { - if info.Digest == "" { - return nil, errors.New(`no digest supplied when reading blob`) - } - if err := info.Digest.Validate(); err != nil { - return nil, perrors.Wrapf(err, `invalid digest supplied when reading blob`) - } - // Assume it's a file, since we're only calling this from a place that expects to read files. - if filename, ok := s.filenames[info.Digest]; ok { - contents, err2 := os.ReadFile(filename) - if err2 != nil { - return nil, perrors.Wrapf(err2, `reading blob from file %q`, filename) - } - return contents, nil - } - // If it's not a file, it's a bug, because we're not expecting to be asked for a layer. - return nil, errors.New("blob not found") -} - -// queueOrCommit queues in the specified blob to be committed to the storage. -// If no other goroutine is already committing layers, the layer and all -// subsequent layers (if already queued) will be committed to the storage. -func (s *storageImageDestination) queueOrCommit(ctx context.Context, blob types.BlobInfo, index int, emptyLayer bool) error { - // NOTE: whenever the code below is touched, make sure that all code - // paths unlock the lock and to unlock it exactly once. - // - // Conceptually, the code is divided in two stages: - // - // 1) Queue in work by marking the layer as ready to be committed. - // If at least one previous/parent layer with a lower index has - // not yet been committed, return early. - // - // 2) Process the queued-in work by committing the "ready" layers - // in sequence. Make sure that more items can be queued-in - // during the comparatively I/O expensive task of committing a - // layer. - // - // The conceptual benefit of this design is that caller can continue - // pulling layers after an early return. At any given time, only one - // caller is the "worker" routine committing layers. All other routines - // can continue pulling and queuing in layers. - s.lock.Lock() - s.indexToPulledLayerInfo[index] = &manifest.LayerInfo{ - BlobInfo: blob, - EmptyLayer: emptyLayer, - } - - // We're still waiting for at least one previous/parent layer to be - // committed, so there's nothing to do. - if index != s.currentIndex { - s.lock.Unlock() - return nil - } - - for info := s.indexToPulledLayerInfo[index]; info != nil; info = s.indexToPulledLayerInfo[index] { - s.lock.Unlock() - // Note: commitLayer locks on-demand. - if err := s.commitLayer(ctx, *info, index); err != nil { - return err - } - s.lock.Lock() - index++ - } - - // Set the index at the very end to make sure that only one routine - // enters stage 2). - s.currentIndex = index - s.lock.Unlock() - return nil -} - -// commitLayer commits the specified blob with the given index to the storage. -// Note that the previous layer is expected to already be committed. -// -// Caution: this function must be called without holding `s.lock`. Callers -// must guarantee that, at any given time, at most one goroutine may execute -// `commitLayer()`. -func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest.LayerInfo, index int) error { - // Already committed? Return early. - if _, alreadyCommitted := s.indexToStorageID[index]; alreadyCommitted { - return nil - } - - // Start with an empty string or the previous layer ID. Note that - // `s.indexToStorageID` can only be accessed by *one* goroutine at any - // given time. Hence, we don't need to lock accesses. - var lastLayer string - if prev := s.indexToStorageID[index-1]; prev != nil { - lastLayer = *prev - } - - // Carry over the previous ID for empty non-base layers. - if blob.EmptyLayer { - s.indexToStorageID[index] = &lastLayer - return nil - } - - // Check if there's already a layer with the ID that we'd give to the result of applying - // this layer blob to its parent, if it has one, or the blob's hex value otherwise. - s.lock.Lock() - diffID, haveDiffID := s.blobDiffIDs[blob.Digest] - s.lock.Unlock() - if !haveDiffID { - // Check if it's elsewhere and the caller just forgot to pass it to us in a PutBlob(), - // or to even check if we had it. - // Use none.NoCache to avoid a repeated DiffID lookup in the BlobInfoCache; a caller - // that relies on using a blob digest that has never been seen by the store had better call - // TryReusingBlob; not calling PutBlob already violates the documented API, so there’s only - // so far we are going to accommodate that (if we should be doing that at all). - logrus.Debugf("looking for diffID for blob %+v", blob.Digest) - // NOTE: use `TryReusingBlob` to prevent recursion. - has, _, err := s.TryReusingBlob(ctx, blob.BlobInfo, none.NoCache, false) - if err != nil { - return perrors.Wrapf(err, "checking for a layer based on blob %q", blob.Digest.String()) - } - if !has { - return fmt.Errorf("error determining uncompressed digest for blob %q", blob.Digest.String()) - } - diffID, haveDiffID = s.blobDiffIDs[blob.Digest] - if !haveDiffID { - return fmt.Errorf("we have blob %q, but don't know its uncompressed digest", blob.Digest.String()) - } - } - id := diffID.Hex() - if lastLayer != "" { - id = digest.Canonical.FromBytes([]byte(lastLayer + "+" + diffID.Hex())).Hex() - } - if layer, err2 := s.imageRef.transport.store.Layer(id); layer != nil && err2 == nil { - // There's already a layer that should have the right contents, just reuse it. - lastLayer = layer.ID - s.indexToStorageID[index] = &lastLayer - return nil - } - - s.lock.Lock() - diffOutput, ok := s.diffOutputs[blob.Digest] - s.lock.Unlock() - if ok { - layer, err := s.imageRef.transport.store.CreateLayer(id, lastLayer, nil, "", false, nil) - if err != nil { - return err - } - - // FIXME: what to do with the uncompressed digest? - diffOutput.UncompressedDigest = blob.Digest - - if err := s.imageRef.transport.store.ApplyDiffFromStagingDirectory(layer.ID, diffOutput.Target, diffOutput, nil); err != nil { - _ = s.imageRef.transport.store.Delete(layer.ID) - return err - } - - s.indexToStorageID[index] = &layer.ID - return nil - } - - s.lock.Lock() - al, ok := s.blobAdditionalLayer[blob.Digest] - s.lock.Unlock() - if ok { - layer, err := al.PutAs(id, lastLayer, nil) - if err != nil && !errors.Is(err, storage.ErrDuplicateID) { - return perrors.Wrapf(err, "failed to put layer from digest and labels") - } - lastLayer = layer.ID - s.indexToStorageID[index] = &lastLayer - return nil - } - - // Check if we previously cached a file with that blob's contents. If we didn't, - // then we need to read the desired contents from a layer. - s.lock.Lock() - filename, ok := s.filenames[blob.Digest] - s.lock.Unlock() - if !ok { - // Try to find the layer with contents matching that blobsum. - layer := "" - layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(diffID) - if err2 == nil && len(layers) > 0 { - layer = layers[0].ID - } else { - layers, err2 = s.imageRef.transport.store.LayersByCompressedDigest(blob.Digest) - if err2 == nil && len(layers) > 0 { - layer = layers[0].ID - } - } - if layer == "" { - return perrors.Wrapf(err2, "locating layer for blob %q", blob.Digest) - } - // Read the layer's contents. - noCompression := archive.Uncompressed - diffOptions := &storage.DiffOptions{ - Compression: &noCompression, - } - diff, err2 := s.imageRef.transport.store.Diff("", layer, diffOptions) - if err2 != nil { - return perrors.Wrapf(err2, "reading layer %q for blob %q", layer, blob.Digest) - } - // Copy the layer diff to a file. Diff() takes a lock that it holds - // until the ReadCloser that it returns is closed, and PutLayer() wants - // the same lock, so the diff can't just be directly streamed from one - // to the other. - filename = s.computeNextBlobCacheFile() - file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600) - if err != nil { - diff.Close() - return perrors.Wrapf(err, "creating temporary file %q", filename) - } - // Copy the data to the file. - // TODO: This can take quite some time, and should ideally be cancellable using - // ctx.Done(). - _, err = io.Copy(file, diff) - diff.Close() - file.Close() - if err != nil { - return perrors.Wrapf(err, "storing blob to file %q", filename) - } - // Make sure that we can find this file later, should we need the layer's - // contents again. - s.lock.Lock() - s.filenames[blob.Digest] = filename - s.lock.Unlock() - } - // Read the cached blob and use it as a diff. - file, err := os.Open(filename) - if err != nil { - return perrors.Wrapf(err, "opening file %q", filename) - } - defer file.Close() - // Build the new layer using the diff, regardless of where it came from. - // TODO: This can take quite some time, and should ideally be cancellable using ctx.Done(). - layer, _, err := s.imageRef.transport.store.PutLayer(id, lastLayer, nil, "", false, &storage.LayerOptions{ - OriginalDigest: blob.Digest, - UncompressedDigest: diffID, - }, file) - if err != nil && !errors.Is(err, storage.ErrDuplicateID) { - return perrors.Wrapf(err, "adding layer with blob %q", blob.Digest) - } - - s.indexToStorageID[index] = &layer.ID - return nil -} - -// Commit marks the process of storing the image as successful and asks for the image to be persisted. -// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list -// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the -// original manifest list digest, if desired. -// WARNING: This does not have any transactional semantics: -// - Uploaded data MAY be visible to others before Commit() is called -// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed) -func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error { - if len(s.manifest) == 0 { - return errors.New("Internal error: storageImageDestination.Commit() called without PutManifest()") - } - toplevelManifest, _, err := unparsedToplevel.Manifest(ctx) - if err != nil { - return perrors.Wrapf(err, "retrieving top-level manifest") - } - // If the name we're saving to includes a digest, then check that the - // manifests that we're about to save all either match the one from the - // unparsedToplevel, or match the digest in the name that we're using. - if s.imageRef.named != nil { - if digested, ok := s.imageRef.named.(reference.Digested); ok { - matches, err := manifest.MatchesDigest(s.manifest, digested.Digest()) - if err != nil { - return err - } - if !matches { - matches, err = manifest.MatchesDigest(toplevelManifest, digested.Digest()) - if err != nil { - return err - } - } - if !matches { - return fmt.Errorf("Manifest to be saved does not match expected digest %s", digested.Digest()) - } - } - } - // Find the list of layer blobs. - man, err := manifest.FromBlob(s.manifest, manifest.GuessMIMEType(s.manifest)) - if err != nil { - return perrors.Wrapf(err, "parsing manifest") - } - layerBlobs := man.LayerInfos() - - // Extract, commit, or find the layers. - for i, blob := range layerBlobs { - if err := s.commitLayer(ctx, blob, i); err != nil { - return err - } - } - var lastLayer string - if len(layerBlobs) > 0 { // Can happen when using caches - prev := s.indexToStorageID[len(layerBlobs)-1] - if prev == nil { - return fmt.Errorf("Internal error: StorageImageDestination.Commit(): previous layer %d hasn't been committed (lastLayer == nil)", len(layerBlobs)-1) - } - lastLayer = *prev - } - - // If one of those blobs was a configuration blob, then we can try to dig out the date when the image - // was originally created, in case we're just copying it. If not, no harm done. - options := &storage.ImageOptions{} - if inspect, err := man.Inspect(s.getConfigBlob); err == nil && inspect.Created != nil { - logrus.Debugf("setting image creation date to %s", inspect.Created) - options.CreationDate = *inspect.Created - } - // Create the image record, pointing to the most-recently added layer. - intendedID := s.imageRef.id - if intendedID == "" { - intendedID = s.computeID(man) - } - oldNames := []string{} - img, err := s.imageRef.transport.store.CreateImage(intendedID, nil, lastLayer, "", options) - if err != nil { - if !errors.Is(err, storage.ErrDuplicateID) { - logrus.Debugf("error creating image: %q", err) - return perrors.Wrapf(err, "creating image %q", intendedID) - } - img, err = s.imageRef.transport.store.Image(intendedID) - if err != nil { - return perrors.Wrapf(err, "reading image %q", intendedID) - } - if img.TopLayer != lastLayer { - logrus.Debugf("error creating image: image with ID %q exists, but uses different layers", intendedID) - return perrors.Wrapf(storage.ErrDuplicateID, "image with ID %q already exists, but uses a different top layer", intendedID) - } - logrus.Debugf("reusing image ID %q", img.ID) - oldNames = append(oldNames, img.Names...) - } else { - logrus.Debugf("created new image ID %q", img.ID) - } - - // Clean up the unfinished image on any error. - // (Is this the right thing to do if the image has existed before?) - commitSucceeded := false - defer func() { - if !commitSucceeded { - logrus.Errorf("Updating image %q (old names %v) failed, deleting it", img.ID, oldNames) - if _, err := s.imageRef.transport.store.DeleteImage(img.ID, true); err != nil { - logrus.Errorf("Error deleting incomplete image %q: %v", img.ID, err) - } - } - }() - - // Add the non-layer blobs as data items. Since we only share layers, they should all be in files, so - // we just need to screen out the ones that are actually layers to get the list of non-layers. - dataBlobs := make(map[digest.Digest]struct{}) - for blob := range s.filenames { - dataBlobs[blob] = struct{}{} - } - for _, layerBlob := range layerBlobs { - delete(dataBlobs, layerBlob.Digest) - } - for blob := range dataBlobs { - v, err := os.ReadFile(s.filenames[blob]) - if err != nil { - return perrors.Wrapf(err, "copying non-layer blob %q to image", blob) - } - if err := s.imageRef.transport.store.SetImageBigData(img.ID, blob.String(), v, manifest.Digest); err != nil { - logrus.Debugf("error saving big data %q for image %q: %v", blob.String(), img.ID, err) - return perrors.Wrapf(err, "saving big data %q for image %q", blob.String(), img.ID) - } - } - // Save the unparsedToplevel's manifest if it differs from the per-platform one, which is saved below. - if len(toplevelManifest) != 0 && !bytes.Equal(toplevelManifest, s.manifest) { - manifestDigest, err := manifest.Digest(toplevelManifest) - if err != nil { - return perrors.Wrapf(err, "digesting top-level manifest") - } - key := manifestBigDataKey(manifestDigest) - if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, toplevelManifest, manifest.Digest); err != nil { - logrus.Debugf("error saving top-level manifest for image %q: %v", img.ID, err) - return perrors.Wrapf(err, "saving top-level manifest for image %q", img.ID) - } - } - // Save the image's manifest. Allow looking it up by digest by using the key convention defined by the Store. - // Record the manifest twice: using a digest-specific key to allow references to that specific digest instance, - // and using storage.ImageDigestBigDataKey for future users that don’t specify any digest and for compatibility with older readers. - key := manifestBigDataKey(s.manifestDigest) - if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil { - logrus.Debugf("error saving manifest for image %q: %v", img.ID, err) - return perrors.Wrapf(err, "saving manifest for image %q", img.ID) - } - key = storage.ImageDigestBigDataKey - if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil { - logrus.Debugf("error saving manifest for image %q: %v", img.ID, err) - return perrors.Wrapf(err, "saving manifest for image %q", img.ID) - } - // Save the signatures, if we have any. - if len(s.signatures) > 0 { - if err := s.imageRef.transport.store.SetImageBigData(img.ID, "signatures", s.signatures, manifest.Digest); err != nil { - logrus.Debugf("error saving signatures for image %q: %v", img.ID, err) - return perrors.Wrapf(err, "saving signatures for image %q", img.ID) - } - } - for instanceDigest, signatures := range s.signatureses { - key := signatureBigDataKey(instanceDigest) - if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, signatures, manifest.Digest); err != nil { - logrus.Debugf("error saving signatures for image %q: %v", img.ID, err) - return perrors.Wrapf(err, "saving signatures for image %q", img.ID) - } - } - // Save our metadata. - metadata, err := json.Marshal(s) - if err != nil { - logrus.Debugf("error encoding metadata for image %q: %v", img.ID, err) - return perrors.Wrapf(err, "encoding metadata for image %q", img.ID) - } - if len(metadata) != 0 { - if err = s.imageRef.transport.store.SetMetadata(img.ID, string(metadata)); err != nil { - logrus.Debugf("error saving metadata for image %q: %v", img.ID, err) - return perrors.Wrapf(err, "saving metadata for image %q", img.ID) - } - logrus.Debugf("saved image metadata %q", string(metadata)) - } - // Adds the reference's name on the image. We don't need to worry about avoiding duplicate - // values because AddNames() will deduplicate the list that we pass to it. - if name := s.imageRef.DockerReference(); name != nil { - if err := s.imageRef.transport.store.AddNames(img.ID, []string{name.String()}); err != nil { - return perrors.Wrapf(err, "adding names %v to image %q", name, img.ID) - } - logrus.Debugf("added name %q to image %q", name, img.ID) - } - - commitSucceeded = true - return nil -} - -// PutManifest writes the manifest to the destination. -func (s *storageImageDestination) PutManifest(ctx context.Context, manifestBlob []byte, instanceDigest *digest.Digest) error { - digest, err := manifest.Digest(manifestBlob) - if err != nil { - return err - } - newBlob := make([]byte, len(manifestBlob)) - copy(newBlob, manifestBlob) - s.manifest = newBlob - s.manifestDigest = digest - return nil -} - -// PutSignatures records the image's signatures for committing as a single data blob. -func (s *storageImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { - sizes := []int{} - sigblob := []byte{} - for _, sig := range signatures { - sizes = append(sizes, len(sig)) - newblob := make([]byte, len(sigblob)+len(sig)) - copy(newblob, sigblob) - copy(newblob[len(sigblob):], sig) - sigblob = newblob - } - if instanceDigest == nil { - s.signatures = sigblob - s.SignatureSizes = sizes - if len(s.manifest) > 0 { - manifestDigest := s.manifestDigest - instanceDigest = &manifestDigest - } - } - if instanceDigest != nil { - s.signatureses[*instanceDigest] = sigblob - s.SignaturesSizes[*instanceDigest] = sizes - } - return nil -} - -// getSize() adds up the sizes of the image's data blobs (which includes the configuration blob), the -// signatures, and the uncompressed sizes of all of the image's layers. -func (s *storageImageSource) getSize() (int64, error) { - var sum int64 - // Size up the data blobs. - dataNames, err := s.imageRef.transport.store.ListImageBigData(s.image.ID) - if err != nil { - return -1, perrors.Wrapf(err, "reading image %q", s.image.ID) - } - for _, dataName := range dataNames { - bigSize, err := s.imageRef.transport.store.ImageBigDataSize(s.image.ID, dataName) - if err != nil { - return -1, perrors.Wrapf(err, "reading data blob size %q for %q", dataName, s.image.ID) - } - sum += bigSize - } - // Add the signature sizes. - for _, sigSize := range s.SignatureSizes { - sum += int64(sigSize) - } - // Walk the layer list. - layerID := s.image.TopLayer - for layerID != "" { - layer, err := s.imageRef.transport.store.Layer(layerID) - if err != nil { - return -1, err - } - if layer.UncompressedDigest == "" || layer.UncompressedSize < 0 { - return -1, fmt.Errorf("size for layer %q is unknown, failing getSize()", layerID) - } - sum += layer.UncompressedSize - if layer.Parent == "" { - break - } - layerID = layer.Parent - } - return sum, nil -} - -// Size() adds up the sizes of the image's data blobs (which includes the configuration blob), the -// signatures, and the uncompressed sizes of all of the image's layers. -func (s *storageImageSource) Size() (int64, error) { - return s.getSize() -} - // Size() returns the previously-computed size of the image, with no error. func (s *storageImageCloser) Size() (int64, error) { return s.size, nil diff --git a/storage/storage_src.go b/storage/storage_src.go new file mode 100644 index 000000000..2ef56b70b --- /dev/null +++ b/storage/storage_src.go @@ -0,0 +1,371 @@ +//go:build !containers_image_storage_stub +// +build !containers_image_storage_stub + +package storage + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "sync" + + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/internal/image" + "github.com/containers/image/v5/internal/imagesource/impl" + "github.com/containers/image/v5/internal/imagesource/stubs" + "github.com/containers/image/v5/internal/tmpdir" + "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/types" + "github.com/containers/storage" + "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/ioutils" + digest "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + perrors "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type storageImageSource struct { + impl.PropertyMethodsInitialize + stubs.NoGetBlobAtInitialize + + imageRef storageReference + image *storage.Image + systemContext *types.SystemContext // SystemContext used in GetBlob() to create temporary files + layerPosition map[digest.Digest]int // Where we are in reading a blob's layers + cachedManifest []byte // A cached copy of the manifest, if already known, or nil + getBlobMutex sync.Mutex // Mutex to sync state for parallel GetBlob executions + SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice + SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // List of sizes of each signature slice +} + +// newImageSource sets up an image for reading. +func newImageSource(ctx context.Context, sys *types.SystemContext, imageRef storageReference) (*storageImageSource, error) { + // First, locate the image. + img, err := imageRef.resolveImage(sys) + if err != nil { + return nil, err + } + + // Build the reader object. + image := &storageImageSource{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + HasThreadSafeGetBlob: true, + }), + NoGetBlobAtInitialize: stubs.NoGetBlobAt(imageRef), + + imageRef: imageRef, + systemContext: sys, + image: img, + layerPosition: make(map[digest.Digest]int), + SignatureSizes: []int{}, + SignaturesSizes: make(map[digest.Digest][]int), + } + if img.Metadata != "" { + if err := json.Unmarshal([]byte(img.Metadata), image); err != nil { + return nil, perrors.Wrap(err, "decoding metadata for source image") + } + } + return image, nil +} + +// Reference returns the image reference that we used to find this image. +func (s *storageImageSource) Reference() types.ImageReference { + return s.imageRef +} + +// Close cleans up any resources we tied up while reading the image. +func (s *storageImageSource) Close() error { + return nil +} + +// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). +// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. +// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. +func (s *storageImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (rc io.ReadCloser, n int64, err error) { + if info.Digest == image.GzippedEmptyLayerDigest { + return io.NopCloser(bytes.NewReader(image.GzippedEmptyLayer)), int64(len(image.GzippedEmptyLayer)), nil + } + + // NOTE: the blob is first written to a temporary file and subsequently + // closed. The intention is to keep the time we own the storage lock + // as short as possible to allow other processes to access the storage. + rc, n, _, err = s.getBlobAndLayerID(info) + if err != nil { + return nil, 0, err + } + defer rc.Close() + + tmpFile, err := os.CreateTemp(tmpdir.TemporaryDirectoryForBigFiles(s.systemContext), "") + if err != nil { + return nil, 0, err + } + + if _, err := io.Copy(tmpFile, rc); err != nil { + return nil, 0, err + } + + if _, err := tmpFile.Seek(0, 0); err != nil { + return nil, 0, err + } + + wrapper := ioutils.NewReadCloserWrapper(tmpFile, func() error { + defer os.Remove(tmpFile.Name()) + return tmpFile.Close() + }) + + return wrapper, n, err +} + +// getBlobAndLayer reads the data blob or filesystem layer which matches the digest and size, if given. +func (s *storageImageSource) getBlobAndLayerID(info types.BlobInfo) (rc io.ReadCloser, n int64, layerID string, err error) { + var layer storage.Layer + var diffOptions *storage.DiffOptions + // We need a valid digest value. + err = info.Digest.Validate() + if err != nil { + return nil, -1, "", err + } + // Check if the blob corresponds to a diff that was used to initialize any layers. Our + // callers should try to retrieve layers using their uncompressed digests, so no need to + // check if they're using one of the compressed digests, which we can't reproduce anyway. + layers, _ := s.imageRef.transport.store.LayersByUncompressedDigest(info.Digest) + + // If it's not a layer, then it must be a data item. + if len(layers) == 0 { + b, err := s.imageRef.transport.store.ImageBigData(s.image.ID, info.Digest.String()) + if err != nil { + return nil, -1, "", err + } + r := bytes.NewReader(b) + logrus.Debugf("exporting opaque data as blob %q", info.Digest.String()) + return io.NopCloser(r), int64(r.Len()), "", nil + } + // Step through the list of matching layers. Tests may want to verify that if we have multiple layers + // which claim to have the same contents, that we actually do have multiple layers, otherwise we could + // just go ahead and use the first one every time. + s.getBlobMutex.Lock() + i := s.layerPosition[info.Digest] + s.layerPosition[info.Digest] = i + 1 + s.getBlobMutex.Unlock() + if len(layers) > 0 { + layer = layers[i%len(layers)] + } + // Force the storage layer to not try to match any compression that was used when the layer was first + // handed to it. + noCompression := archive.Uncompressed + diffOptions = &storage.DiffOptions{ + Compression: &noCompression, + } + if layer.UncompressedSize < 0 { + n = -1 + } else { + n = layer.UncompressedSize + } + logrus.Debugf("exporting filesystem layer %q without compression for blob %q", layer.ID, info.Digest) + rc, err = s.imageRef.transport.store.Diff("", layer.ID, diffOptions) + if err != nil { + return nil, -1, "", err + } + return rc, n, layer.ID, err +} + +// GetManifest() reads the image's manifest. +func (s *storageImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) (manifestBlob []byte, MIMEType string, err error) { + if instanceDigest != nil { + key := manifestBigDataKey(*instanceDigest) + blob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key) + if err != nil { + return nil, "", perrors.Wrapf(err, "reading manifest for image instance %q", *instanceDigest) + } + return blob, manifest.GuessMIMEType(blob), err + } + if len(s.cachedManifest) == 0 { + // The manifest is stored as a big data item. + // Prefer the manifest corresponding to the user-specified digest, if available. + if s.imageRef.named != nil { + if digested, ok := s.imageRef.named.(reference.Digested); ok { + key := manifestBigDataKey(digested.Digest()) + blob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key) + if err != nil && !os.IsNotExist(err) { // os.IsNotExist is true if the image exists but there is no data corresponding to key + return nil, "", err + } + if err == nil { + s.cachedManifest = blob + } + } + } + // If the user did not specify a digest, or this is an old image stored before manifestBigDataKey was introduced, use the default manifest. + // Note that the manifest may not match the expected digest, and that is likely to fail eventually, e.g. in c/image/image/UnparsedImage.Manifest(). + if len(s.cachedManifest) == 0 { + cachedBlob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, storage.ImageDigestBigDataKey) + if err != nil { + return nil, "", err + } + s.cachedManifest = cachedBlob + } + } + return s.cachedManifest, manifest.GuessMIMEType(s.cachedManifest), err +} + +// LayerInfosForCopy() returns the list of layer blobs that make up the root filesystem of +// the image, after they've been decompressed. +func (s *storageImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) { + manifestBlob, manifestType, err := s.GetManifest(ctx, instanceDigest) + if err != nil { + return nil, perrors.Wrapf(err, "reading image manifest for %q", s.image.ID) + } + if manifest.MIMETypeIsMultiImage(manifestType) { + return nil, errors.New("can't copy layers for a manifest list (shouldn't be attempted)") + } + man, err := manifest.FromBlob(manifestBlob, manifestType) + if err != nil { + return nil, perrors.Wrapf(err, "parsing image manifest for %q", s.image.ID) + } + + uncompressedLayerType := "" + switch manifestType { + case imgspecv1.MediaTypeImageManifest: + uncompressedLayerType = imgspecv1.MediaTypeImageLayer + case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema2MediaType: + uncompressedLayerType = manifest.DockerV2SchemaLayerMediaTypeUncompressed + } + + physicalBlobInfos := []types.BlobInfo{} + layerID := s.image.TopLayer + for layerID != "" { + layer, err := s.imageRef.transport.store.Layer(layerID) + if err != nil { + return nil, perrors.Wrapf(err, "reading layer %q in image %q", layerID, s.image.ID) + } + if layer.UncompressedDigest == "" { + return nil, fmt.Errorf("uncompressed digest for layer %q is unknown", layerID) + } + if layer.UncompressedSize < 0 { + return nil, fmt.Errorf("uncompressed size for layer %q is unknown", layerID) + } + blobInfo := types.BlobInfo{ + Digest: layer.UncompressedDigest, + Size: layer.UncompressedSize, + MediaType: uncompressedLayerType, + } + physicalBlobInfos = append([]types.BlobInfo{blobInfo}, physicalBlobInfos...) + layerID = layer.Parent + } + + res, err := buildLayerInfosForCopy(man.LayerInfos(), physicalBlobInfos) + if err != nil { + return nil, perrors.Wrapf(err, "creating LayerInfosForCopy of image %q", s.image.ID) + } + return res, nil +} + +// buildLayerInfosForCopy builds a LayerInfosForCopy return value based on manifestInfos from the original manifest, +// but using layer data which we can actually produce — physicalInfos for non-empty layers, +// and image.GzippedEmptyLayer for empty ones. +// (This is split basically only to allow easily unit-testing the part that has no dependencies on the external environment.) +func buildLayerInfosForCopy(manifestInfos []manifest.LayerInfo, physicalInfos []types.BlobInfo) ([]types.BlobInfo, error) { + nextPhysical := 0 + res := make([]types.BlobInfo, len(manifestInfos)) + for i, mi := range manifestInfos { + if mi.EmptyLayer { + res[i] = types.BlobInfo{ + Digest: image.GzippedEmptyLayerDigest, + Size: int64(len(image.GzippedEmptyLayer)), + MediaType: mi.MediaType, + } + } else { + if nextPhysical >= len(physicalInfos) { + return nil, fmt.Errorf("expected more than %d physical layers to exist", len(physicalInfos)) + } + res[i] = physicalInfos[nextPhysical] + nextPhysical++ + } + } + if nextPhysical != len(physicalInfos) { + return nil, fmt.Errorf("used only %d out of %d physical layers", nextPhysical, len(physicalInfos)) + } + return res, nil +} + +// GetSignatures() parses the image's signatures blob into a slice of byte slices. +func (s *storageImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) (signatures [][]byte, err error) { + var offset int + sigslice := [][]byte{} + signature := []byte{} + signatureSizes := s.SignatureSizes + key := "signatures" + instance := "default instance" + if instanceDigest != nil { + signatureSizes = s.SignaturesSizes[*instanceDigest] + key = signatureBigDataKey(*instanceDigest) + instance = instanceDigest.Encoded() + } + if len(signatureSizes) > 0 { + signatureBlob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key) + if err != nil { + return nil, perrors.Wrapf(err, "looking up signatures data for image %q (%s)", s.image.ID, instance) + } + signature = signatureBlob + } + for _, length := range signatureSizes { + if offset+length > len(signature) { + return nil, perrors.Wrapf(err, "looking up signatures data for image %q (%s): expected at least %d bytes, only found %d", s.image.ID, instance, len(signature), offset+length) + } + sigslice = append(sigslice, signature[offset:offset+length]) + offset += length + } + if offset != len(signature) { + return nil, fmt.Errorf("signatures data (%s) contained %d extra bytes", instance, len(signatures)-offset) + } + return sigslice, nil +} + +// getSize() adds up the sizes of the image's data blobs (which includes the configuration blob), the +// signatures, and the uncompressed sizes of all of the image's layers. +func (s *storageImageSource) getSize() (int64, error) { + var sum int64 + // Size up the data blobs. + dataNames, err := s.imageRef.transport.store.ListImageBigData(s.image.ID) + if err != nil { + return -1, perrors.Wrapf(err, "reading image %q", s.image.ID) + } + for _, dataName := range dataNames { + bigSize, err := s.imageRef.transport.store.ImageBigDataSize(s.image.ID, dataName) + if err != nil { + return -1, perrors.Wrapf(err, "reading data blob size %q for %q", dataName, s.image.ID) + } + sum += bigSize + } + // Add the signature sizes. + for _, sigSize := range s.SignatureSizes { + sum += int64(sigSize) + } + // Walk the layer list. + layerID := s.image.TopLayer + for layerID != "" { + layer, err := s.imageRef.transport.store.Layer(layerID) + if err != nil { + return -1, err + } + if layer.UncompressedDigest == "" || layer.UncompressedSize < 0 { + return -1, fmt.Errorf("size for layer %q is unknown, failing getSize()", layerID) + } + sum += layer.UncompressedSize + if layer.Parent == "" { + break + } + layerID = layer.Parent + } + return sum, nil +} + +// Size() adds up the sizes of the image's data blobs (which includes the configuration blob), the +// signatures, and the uncompressed sizes of all of the image's layers. +func (s *storageImageSource) Size() (int64, error) { + return s.getSize() +} diff --git a/storage/storage_image_test.go b/storage/storage_src_test.go similarity index 100% rename from storage/storage_image_test.go rename to storage/storage_src_test.go diff --git a/storage/storage_test.go b/storage/storage_test.go index ef69399da..6bbab68e4 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -39,6 +39,7 @@ var ( _ types.ImageDestination = &storageImageDestination{} _ private.ImageDestination = (*storageImageDestination)(nil) _ types.ImageSource = &storageImageSource{} + _ private.ImageSource = (*storageImageSource)(nil) _ types.ImageReference = &storageReference{} _ types.ImageTransport = &storageTransport{} ) diff --git a/tarball/tarball_src.go b/tarball/tarball_src.go index aedfdf5de..039df406c 100644 --- a/tarball/tarball_src.go +++ b/tarball/tarball_src.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "github.com/containers/image/v5/internal/imagesource/impl" + "github.com/containers/image/v5/internal/imagesource/stubs" "github.com/containers/image/v5/types" "github.com/klauspost/pgzip" digest "github.com/opencontainers/go-digest" @@ -19,6 +21,11 @@ import ( ) type tarballImageSource struct { + impl.PropertyMethodsInitialize + impl.NoSignatures + impl.DoesNotAffectLayerInfosForCopy + stubs.NoGetBlobAtInitialize + reference tarballReference filenames []string diffIDs []digest.Digest @@ -185,6 +192,11 @@ func (r *tarballReference) NewImageSource(ctx context.Context, sys *types.System // Return the image. src := &tarballImageSource{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + HasThreadSafeGetBlob: false, + }), + NoGetBlobAtInitialize: stubs.NoGetBlobAt(r), + reference: *r, filenames: filenames, diffIDs: diffIDs, @@ -205,11 +217,6 @@ func (is *tarballImageSource) Close() error { return nil } -// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. -func (is *tarballImageSource) HasThreadSafeGetBlob() bool { - return false -} - // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. @@ -246,28 +253,6 @@ func (is *tarballImageSource) GetManifest(ctx context.Context, instanceDigest *d return is.manifest, imgspecv1.MediaTypeImageManifest, nil } -// GetSignatures returns the image's signatures. It may use a remote (= slow) service. -// This source implementation does not support manifest lists, so the passed-in instanceDigest should always be nil, -// as there can be no secondary manifests. -func (*tarballImageSource) GetSignatures(ctx context.Context, instanceDigest *digest.Digest) ([][]byte, error) { - if instanceDigest != nil { - return nil, fmt.Errorf("manifest lists are not supported by the %q transport", transportName) - } - return nil, nil -} - func (is *tarballImageSource) Reference() types.ImageReference { return &is.reference } - -// LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer -// blobsums that are listed in the image's manifest. If values are returned, they should be used when using GetBlob() -// to read the image's layers. -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve BlobInfos for -// (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list -// (e.g. if the source never returns manifest lists). -// The Digest field is guaranteed to be provided; Size may be -1. -// WARNING: The list may contain duplicates, and they are semantically relevant. -func (*tarballImageSource) LayerInfosForCopy(context.Context, *digest.Digest) ([]types.BlobInfo, error) { - return nil, nil -} diff --git a/tarball/tarball_src_test.go b/tarball/tarball_src_test.go new file mode 100644 index 000000000..5d9f48cf6 --- /dev/null +++ b/tarball/tarball_src_test.go @@ -0,0 +1,5 @@ +package tarball + +import "github.com/containers/image/v5/internal/private" + +var _ private.ImageSource = (*tarballImageSource)(nil)