diff --git a/directory/directory_dest.go b/directory/directory_dest.go index 50c52879e..8215c9089 100644 --- a/directory/directory_dest.go +++ b/directory/directory_dest.go @@ -9,6 +9,9 @@ import ( "path/filepath" "runtime" + "github.com/containers/image/v5/internal/imagedestination/impl" + "github.com/containers/image/v5/internal/imagedestination/stubs" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/internal/putblobdigest" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" @@ -23,12 +26,16 @@ const version = "Directory Transport Version: 1.1\n" var ErrNotContainerImageDir = errors.New("not a containers image directory, don't want to overwrite important data") type dirImageDestination struct { - ref dirReference - desiredLayerCompression types.LayerCompression + impl.Compat + impl.PropertyMethodsInitialize + stubs.NoPutBlobPartialInitialize + stubs.AlwaysSupportsSignatures + + ref dirReference } // newImageDestination returns an ImageDestination for writing to a directory. -func newImageDestination(sys *types.SystemContext, ref dirReference) (types.ImageDestination, error) { +func newImageDestination(sys *types.SystemContext, ref dirReference) (private.ImageDestination, error) { desiredLayerCompression := types.PreserveOriginal if sys != nil { if sys.DirForceCompress { @@ -42,28 +49,27 @@ func newImageDestination(sys *types.SystemContext, ref dirReference) (types.Imag desiredLayerCompression = types.Decompress } } - d := &dirImageDestination{ref: ref, desiredLayerCompression: desiredLayerCompression} // If directory exists check if it is empty // if not empty, check whether the contents match that of a container image directory and overwrite the contents // if the contents don't match throw an error - dirExists, err := pathExists(d.ref.resolvedPath) + dirExists, err := pathExists(ref.resolvedPath) if err != nil { - return nil, perrors.Wrapf(err, "checking for path %q", d.ref.resolvedPath) + return nil, perrors.Wrapf(err, "checking for path %q", ref.resolvedPath) } if dirExists { - isEmpty, err := isDirEmpty(d.ref.resolvedPath) + isEmpty, err := isDirEmpty(ref.resolvedPath) if err != nil { return nil, err } if !isEmpty { - versionExists, err := pathExists(d.ref.versionPath()) + versionExists, err := pathExists(ref.versionPath()) if err != nil { - return nil, perrors.Wrapf(err, "checking if path exists %q", d.ref.versionPath()) + return nil, perrors.Wrapf(err, "checking if path exists %q", ref.versionPath()) } if versionExists { - contents, err := os.ReadFile(d.ref.versionPath()) + contents, err := os.ReadFile(ref.versionPath()) if err != nil { return nil, err } @@ -75,22 +81,37 @@ func newImageDestination(sys *types.SystemContext, ref dirReference) (types.Imag return nil, ErrNotContainerImageDir } // delete directory contents so that only one image is in the directory at a time - if err = removeDirContents(d.ref.resolvedPath); err != nil { - return nil, perrors.Wrapf(err, "erasing contents in %q", d.ref.resolvedPath) + if err = removeDirContents(ref.resolvedPath); err != nil { + return nil, perrors.Wrapf(err, "erasing contents in %q", ref.resolvedPath) } - logrus.Debugf("overwriting existing container image directory %q", d.ref.resolvedPath) + logrus.Debugf("overwriting existing container image directory %q", ref.resolvedPath) } } else { // create directory if it doesn't exist - if err := os.MkdirAll(d.ref.resolvedPath, 0755); err != nil { - return nil, perrors.Wrapf(err, "unable to create directory %q", d.ref.resolvedPath) + if err := os.MkdirAll(ref.resolvedPath, 0755); err != nil { + return nil, perrors.Wrapf(err, "unable to create directory %q", ref.resolvedPath) } } // create version file - err = os.WriteFile(d.ref.versionPath(), []byte(version), 0644) + err = os.WriteFile(ref.versionPath(), []byte(version), 0644) if err != nil { - return nil, perrors.Wrapf(err, "creating version file %q", d.ref.versionPath()) + return nil, perrors.Wrapf(err, "creating version file %q", ref.versionPath()) } + + d := &dirImageDestination{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + SupportedManifestMIMETypes: nil, + DesiredLayerCompression: desiredLayerCompression, + AcceptsForeignLayerURLs: false, + MustMatchRuntimeOS: false, + IgnoresEmbeddedDockerReference: false, // N/A, DockerReference() returns nil. + HasThreadSafePutBlob: false, + }), + NoPutBlobPartialInitialize: stubs.NoPutBlobPartial(ref), + + ref: ref, + } + d.Compat = impl.AddCompat(d) return d, nil } @@ -105,51 +126,14 @@ func (d *dirImageDestination) Close() error { return nil } -func (d *dirImageDestination) SupportedManifestMIMETypes() []string { - return nil -} - -// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures. -// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil. -func (d *dirImageDestination) SupportsSignatures(ctx context.Context) error { - return nil -} - -func (d *dirImageDestination) DesiredLayerCompression() types.LayerCompression { - return d.desiredLayerCompression -} - -// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually -// uploaded to the image destination, true otherwise. -func (d *dirImageDestination) AcceptsForeignLayerURLs() bool { - return false -} - -// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise. -func (d *dirImageDestination) MustMatchRuntimeOS() bool { - return false -} - -// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(), -// and would prefer to receive an unmodified manifest instead of one modified for the destination. -// Does not make a difference if Reference().DockerReference() is nil. -func (d *dirImageDestination) IgnoresEmbeddedDockerReference() bool { - return false // N/A, DockerReference() returns nil. -} - -// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. -func (d *dirImageDestination) HasThreadSafePutBlob() bool { - return false -} - -// PutBlob writes contents of stream and returns data representing the result (with all data filled in). +// PutBlobWithOptions writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. // inputInfo.Size is the expected length of stream, if known. -// May update cache. +// inputInfo.MediaType describes the blob format, if known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (d *dirImageDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) { +func (d *dirImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options private.PutBlobOptions) (types.BlobInfo, error) { blobFile, err := os.CreateTemp(d.ref.path, "dir-put-blob") if err != nil { return types.BlobInfo{}, err @@ -200,16 +184,14 @@ func (d *dirImageDestination) PutBlob(ctx context.Context, stream io.Reader, inp return types.BlobInfo{Digest: blobDigest, Size: size}, nil } -// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination +// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). // info.Digest must not be empty. -// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input. // If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may // include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be // reflected in the manifest that will be written. // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. -// May use and/or update cache. -func (d *dirImageDestination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) { +func (d *dirImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { if info.Digest == "" { return false, types.BlobInfo{}, fmt.Errorf("Can not check for a blob with unknown digest") } diff --git a/directory/directory_test.go b/directory/directory_test.go index 8c5750af8..c75ebf13c 100644 --- a/directory/directory_test.go +++ b/directory/directory_test.go @@ -8,6 +8,7 @@ import ( "os" "testing" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/blobinfocache/memory" "github.com/containers/image/v5/types" @@ -16,6 +17,8 @@ import ( "github.com/stretchr/testify/require" ) +var _ private.ImageDestination = (*dirImageDestination)(nil) + func TestDestinationReference(t *testing.T) { ref, tmpDir := refToTempDir(t) diff --git a/docker/archive/dest.go b/docker/archive/dest.go index 392ab4f31..60521662e 100644 --- a/docker/archive/dest.go +++ b/docker/archive/dest.go @@ -6,6 +6,7 @@ import ( "io" "github.com/containers/image/v5/docker/internal/tarfile" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/types" ) @@ -16,7 +17,7 @@ type archiveImageDestination struct { writer io.Closer // May be nil if the archive is shared } -func newImageDestination(sys *types.SystemContext, ref archiveReference) (types.ImageDestination, error) { +func newImageDestination(sys *types.SystemContext, ref archiveReference) (private.ImageDestination, error) { if ref.sourceIndex != -1 { return nil, fmt.Errorf("Destination reference must not contain a manifest index @%d", ref.sourceIndex) } @@ -35,7 +36,7 @@ func newImageDestination(sys *types.SystemContext, ref archiveReference) (types. archive = tarfile.NewWriter(fh) writer = fh } - tarDest := tarfile.NewDestination(sys, archive, ref.ref) + tarDest := tarfile.NewDestination(sys, archive, ref.Transport().Name(), ref.ref) if sys != nil && sys.DockerArchiveAdditionalTags != nil { tarDest.AddRepoTags(sys.DockerArchiveAdditionalTags) } @@ -47,11 +48,6 @@ func newImageDestination(sys *types.SystemContext, ref archiveReference) (types. }, nil } -// DesiredLayerCompression indicates if layers must be compressed, decompressed or preserved -func (d *archiveImageDestination) DesiredLayerCompression() types.LayerCompression { - return types.Decompress -} - // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, // e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects. func (d *archiveImageDestination) Reference() types.ImageReference { diff --git a/docker/archive/dest_test.go b/docker/archive/dest_test.go new file mode 100644 index 000000000..06b390931 --- /dev/null +++ b/docker/archive/dest_test.go @@ -0,0 +1,5 @@ +package archive + +import "github.com/containers/image/v5/internal/private" + +var _ private.ImageDestination = (*archiveImageDestination)(nil) diff --git a/docker/daemon/daemon_dest.go b/docker/daemon/daemon_dest.go index 74299a28e..a5914f61f 100644 --- a/docker/daemon/daemon_dest.go +++ b/docker/daemon/daemon_dest.go @@ -8,6 +8,7 @@ import ( "github.com/containers/image/v5/docker/internal/tarfile" "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/types" "github.com/docker/docker/client" perrors "github.com/pkg/errors" @@ -28,7 +29,7 @@ type daemonImageDestination struct { } // newImageDestination returns a types.ImageDestination for the specified image reference. -func newImageDestination(ctx context.Context, sys *types.SystemContext, ref daemonReference) (types.ImageDestination, error) { +func newImageDestination(ctx context.Context, sys *types.SystemContext, ref daemonReference) (private.ImageDestination, error) { if ref.ref == nil { return nil, fmt.Errorf("Invalid destination docker-daemon:%s: a destination must be a name:tag", ref.StringWithinTransport()) } @@ -58,7 +59,7 @@ func newImageDestination(ctx context.Context, sys *types.SystemContext, ref daem return &daemonImageDestination{ ref: ref, mustMatchRuntimeOS: mustMatchRuntimeOS, - Destination: tarfile.NewDestination(sys, archive, namedTaggedRef), + Destination: tarfile.NewDestination(sys, archive, ref.Transport().Name(), namedTaggedRef), archive: archive, goroutineCancel: goroutineCancel, statusChannel: statusChannel, diff --git a/docker/daemon/daemon_dest_test.go b/docker/daemon/daemon_dest_test.go new file mode 100644 index 000000000..51e5d09da --- /dev/null +++ b/docker/daemon/daemon_dest_test.go @@ -0,0 +1,5 @@ +package daemon + +import "github.com/containers/image/v5/internal/private" + +var _ private.ImageDestination = (*daemonImageDestination)(nil) diff --git a/docker/docker_image_dest.go b/docker/docker_image_dest.go index e670c01c4..2c55ac4fb 100644 --- a/docker/docker_image_dest.go +++ b/docker/docker_image_dest.go @@ -16,6 +16,9 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/internal/blobinfocache" + "github.com/containers/image/v5/internal/imagedestination/impl" + "github.com/containers/image/v5/internal/imagedestination/stubs" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/internal/putblobdigest" "github.com/containers/image/v5/internal/streamdigest" "github.com/containers/image/v5/internal/uploadreader" @@ -30,6 +33,10 @@ import ( ) type dockerImageDestination struct { + impl.Compat + impl.PropertyMethodsInitialize + stubs.NoPutBlobPartialInitialize + ref dockerReference c *dockerClient // State @@ -37,15 +44,36 @@ type dockerImageDestination struct { } // newImageDestination creates a new ImageDestination for the specified image reference. -func newImageDestination(sys *types.SystemContext, ref dockerReference) (types.ImageDestination, error) { +func newImageDestination(sys *types.SystemContext, ref dockerReference) (private.ImageDestination, error) { c, err := newDockerClientFromRef(sys, ref, true, "pull,push") if err != nil { return nil, err } - return &dockerImageDestination{ + mimeTypes := []string{ + imgspecv1.MediaTypeImageManifest, + manifest.DockerV2Schema2MediaType, + imgspecv1.MediaTypeImageIndex, + manifest.DockerV2ListMediaType, + } + if c.sys == nil || !c.sys.DockerDisableDestSchema1MIMETypes { + mimeTypes = append(mimeTypes, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema1MediaType) + } + + dest := &dockerImageDestination{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + SupportedManifestMIMETypes: mimeTypes, + DesiredLayerCompression: types.Compress, + MustMatchRuntimeOS: false, + IgnoresEmbeddedDockerReference: false, // We do want the manifest updated; older registry versions refuse manifests if the embedded reference does not match. + HasThreadSafePutBlob: true, + }), + NoPutBlobPartialInitialize: stubs.NoPutBlobPartial(ref), + ref: ref, c: c, - }, nil + } + dest.Compat = impl.AddCompat(dest) + return dest, nil } // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, @@ -59,19 +87,6 @@ func (d *dockerImageDestination) Close() error { return nil } -func (d *dockerImageDestination) SupportedManifestMIMETypes() []string { - mimeTypes := []string{ - imgspecv1.MediaTypeImageManifest, - manifest.DockerV2Schema2MediaType, - imgspecv1.MediaTypeImageIndex, - manifest.DockerV2ListMediaType, - } - if d.c.sys == nil || !d.c.sys.DockerDisableDestSchema1MIMETypes { - mimeTypes = append(mimeTypes, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema1MediaType) - } - return mimeTypes -} - // SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures. // Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil. func (d *dockerImageDestination) SupportsSignatures(ctx context.Context) error { @@ -88,28 +103,12 @@ func (d *dockerImageDestination) SupportsSignatures(ctx context.Context) error { } } -func (d *dockerImageDestination) DesiredLayerCompression() types.LayerCompression { - return types.Compress -} - // AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually // uploaded to the image destination, true otherwise. func (d *dockerImageDestination) AcceptsForeignLayerURLs() bool { return true } -// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise. -func (d *dockerImageDestination) MustMatchRuntimeOS() bool { - return false -} - -// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(), -// and would prefer to receive an unmodified manifest instead of one modified for the destination. -// Does not make a difference if Reference().DockerReference() is nil. -func (d *dockerImageDestination) IgnoresEmbeddedDockerReference() bool { - return false // We do want the manifest updated; older registry versions refuse manifests if the embedded reference does not match. -} - // sizeCounter is an io.Writer which only counts the total size of its input. type sizeCounter struct{ size int64 } @@ -118,19 +117,14 @@ func (c *sizeCounter) Write(p []byte) (n int, err error) { return len(p), nil } -// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. -func (d *dockerImageDestination) HasThreadSafePutBlob() bool { - return true -} - -// PutBlob writes contents of stream and returns data representing the result (with all data filled in). +// PutBlobWithOptions writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. // inputInfo.Size is the expected length of stream, if known. -// May update cache. +// inputInfo.MediaType describes the blob format, if known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (d *dockerImageDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) { +func (d *dockerImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options private.PutBlobOptions) (types.BlobInfo, error) { // If requested, precompute the blob digest to prevent uploading layers that already exist on the registry. // This functionality is particularly useful when BlobInfoCache has not been populated with compressed digests, // the source blob is uncompressed, and the destination blob is being compressed "on the fly". @@ -147,7 +141,7 @@ func (d *dockerImageDestination) PutBlob(ctx context.Context, stream io.Reader, if inputInfo.Digest != "" { // This should not really be necessary, at least the copy code calls TryReusingBlob automatically. // Still, we need to check, if only because the "initiate upload" endpoint does not have a documented "blob already exists" return value. - haveBlob, reusedInfo, err := d.tryReusingExactBlob(ctx, inputInfo, cache) + haveBlob, reusedInfo, err := d.tryReusingExactBlob(ctx, inputInfo, options.Cache) if err != nil { return types.BlobInfo{}, err } @@ -218,7 +212,7 @@ func (d *dockerImageDestination) PutBlob(ctx context.Context, stream io.Reader, } logrus.Debugf("Upload of layer %s complete", blobDigest) - cache.RecordKnownLocation(d.ref.Transport(), bicTransportScope(d.ref), blobDigest, newBICLocationReference(d.ref)) + options.Cache.RecordKnownLocation(d.ref.Transport(), bicTransportScope(d.ref), blobDigest, newBICLocationReference(d.ref)) return types.BlobInfo{Digest: blobDigest, Size: sizeCounter.size}, nil } @@ -296,7 +290,7 @@ func (d *dockerImageDestination) mountBlob(ctx context.Context, srcRepo referenc // tryReusingExactBlob is a subset of TryReusingBlob which _only_ looks for exactly the specified // blob in the current repository, with no cross-repo reuse or mounting; cache may be updated, it is not read. // The caller must ensure info.Digest is set. -func (d *dockerImageDestination) tryReusingExactBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (bool, types.BlobInfo, error) { +func (d *dockerImageDestination) tryReusingExactBlob(ctx context.Context, info types.BlobInfo, cache blobinfocache.BlobInfoCache2) (bool, types.BlobInfo, error) { exists, size, err := d.blobExists(ctx, d.ref.ref, info.Digest, nil) if err != nil { return false, types.BlobInfo{}, err @@ -308,22 +302,20 @@ func (d *dockerImageDestination) tryReusingExactBlob(ctx context.Context, info t return false, types.BlobInfo{}, nil } -// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination +// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). // info.Digest must not be empty. -// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input. // If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may // include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be // reflected in the manifest that will be written. // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. -// May use and/or update cache. -func (d *dockerImageDestination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) { +func (d *dockerImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { if info.Digest == "" { return false, types.BlobInfo{}, errors.New("Can not check for a blob with unknown digest") } // First, check whether the blob happens to already exist at the destination. - haveBlob, reusedInfo, err := d.tryReusingExactBlob(ctx, info, cache) + haveBlob, reusedInfo, err := d.tryReusingExactBlob(ctx, info, options.Cache) if err != nil { return false, types.BlobInfo{}, err } @@ -332,8 +324,7 @@ func (d *dockerImageDestination) TryReusingBlob(ctx context.Context, info types. } // Then try reusing blobs from other locations. - bic := blobinfocache.FromBlobInfoCache(cache) - candidates := bic.CandidateLocations2(d.ref.Transport(), bicTransportScope(d.ref), info.Digest, canSubstitute) + candidates := options.Cache.CandidateLocations2(d.ref.Transport(), bicTransportScope(d.ref), info.Digest, options.CanSubstitute) for _, candidate := range candidates { candidateRepo, err := parseBICLocationReference(candidate.Location) if err != nil { @@ -387,7 +378,7 @@ func (d *dockerImageDestination) TryReusingBlob(ctx context.Context, info types. } } - bic.RecordKnownLocation(d.ref.Transport(), bicTransportScope(d.ref), candidate.Digest, newBICLocationReference(d.ref)) + options.Cache.RecordKnownLocation(d.ref.Transport(), bicTransportScope(d.ref), candidate.Digest, newBICLocationReference(d.ref)) compressionOperation, compressionAlgorithm, err := blobinfocache.OperationAndAlgorithmForCompressor(candidate.CompressorName) if err != nil { diff --git a/docker/docker_image_dest_test.go b/docker/docker_image_dest_test.go index b4baf0fdf..4238d561b 100644 --- a/docker/docker_image_dest_test.go +++ b/docker/docker_image_dest_test.go @@ -6,10 +6,13 @@ import ( "net/http" "testing" + "github.com/containers/image/v5/internal/private" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +var _ private.ImageDestination = (*dockerImageDestination)(nil) + func TestIsManifestInvalidError(t *testing.T) { // Sadly only a smoke test; this really should record all known errors exactly as they happen. diff --git a/docker/internal/tarfile/dest.go b/docker/internal/tarfile/dest.go index 9de7a1b6b..b36f303a3 100644 --- a/docker/internal/tarfile/dest.go +++ b/docker/internal/tarfile/dest.go @@ -8,7 +8,10 @@ import ( "io" "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/internal/imagedestination/impl" + "github.com/containers/image/v5/internal/imagedestination/stubs" "github.com/containers/image/v5/internal/iolimits" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/internal/streamdigest" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/types" @@ -17,8 +20,13 @@ import ( "github.com/sirupsen/logrus" ) -// Destination is a partial implementation of types.ImageDestination for writing to an io.Writer. +// Destination is a partial implementation of private.ImageDestination for writing to an io.Writer. type Destination struct { + impl.Compat + impl.PropertyMethodsInitialize + stubs.NoPutBlobPartialInitialize + stubs.NoSignaturesInitialize + archive *Writer repoTags []reference.NamedTagged // Other state. @@ -27,16 +35,34 @@ type Destination struct { } // NewDestination returns a tarfile.Destination adding images to the specified Writer. -func NewDestination(sys *types.SystemContext, archive *Writer, ref reference.NamedTagged) *Destination { +func NewDestination(sys *types.SystemContext, archive *Writer, transportName string, ref reference.NamedTagged) *Destination { repoTags := []reference.NamedTagged{} if ref != nil { repoTags = append(repoTags, ref) } - return &Destination{ + dest := &Destination{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + SupportedManifestMIMETypes: []string{ + manifest.DockerV2Schema2MediaType, // We rely on the types.Image.UpdatedImage schema conversion capabilities. + }, + DesiredLayerCompression: types.Decompress, + AcceptsForeignLayerURLs: false, + MustMatchRuntimeOS: false, + IgnoresEmbeddedDockerReference: false, // N/A, we only accept schema2 images where EmbeddedDockerReferenceConflicts() is always false. + // The code _is_ actually thread-safe, but apart from computing sizes/digests of layers where + // this is unknown in advance, the actual copy is serialized by d.archive, so there probably isn’t + // much benefit from concurrency, mostly just extra CPU, memory and I/O contention. + HasThreadSafePutBlob: false, + }), + NoPutBlobPartialInitialize: stubs.NoPutBlobPartialRaw(transportName), + NoSignaturesInitialize: stubs.NoSignatures("Storing signatures for docker tar files is not supported"), + archive: archive, repoTags: repoTags, sysCtx: sys, } + dest.Compat = impl.AddCompat(dest) + return dest } // AddRepoTags adds the specified tags to the destination's repoTags. @@ -44,54 +70,14 @@ func (d *Destination) AddRepoTags(tags []reference.NamedTagged) { d.repoTags = append(d.repoTags, tags...) } -// SupportedManifestMIMETypes tells which manifest mime types the destination supports -// If an empty slice or nil it's returned, then any mime type can be tried to upload -func (d *Destination) SupportedManifestMIMETypes() []string { - return []string{ - manifest.DockerV2Schema2MediaType, // We rely on the types.Image.UpdatedImage schema conversion capabilities. - } -} - -// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures. -// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil. -func (d *Destination) SupportsSignatures(ctx context.Context) error { - return errors.New("Storing signatures for docker tar files is not supported") -} - -// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually -// uploaded to the image destination, true otherwise. -func (d *Destination) AcceptsForeignLayerURLs() bool { - return false -} - -// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise. -func (d *Destination) MustMatchRuntimeOS() bool { - return false -} - -// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(), -// and would prefer to receive an unmodified manifest instead of one modified for the destination. -// Does not make a difference if Reference().DockerReference() is nil. -func (d *Destination) IgnoresEmbeddedDockerReference() bool { - return false // N/A, we only accept schema2 images where EmbeddedDockerReferenceConflicts() is always false. -} - -// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. -func (d *Destination) HasThreadSafePutBlob() bool { - // The code _is_ actually thread-safe, but apart from computing sizes/digests of layers where - // this is unknown in advance, the actual copy is serialized by d.archive, so there probably isn’t - // much benefit from concurrency, mostly just extra CPU, memory and I/O contention. - return false -} - -// PutBlob writes contents of stream and returns data representing the result (with all data filled in). +// PutBlobWithOptions writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. // inputInfo.Size is the expected length of stream, if known. -// May update cache. +// inputInfo.MediaType describes the blob format, if known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (d *Destination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) { +func (d *Destination) PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options private.PutBlobOptions) (types.BlobInfo, error) { // Ouch, we need to stream the blob into a temporary file just to determine the size. // When the layer is decompressed, we also have to generate the digest on uncompressed data. if inputInfo.Size == -1 || inputInfo.Digest == "" { @@ -119,7 +105,7 @@ func (d *Destination) PutBlob(ctx context.Context, stream io.Reader, inputInfo t return reusedInfo, nil } - if isConfig { + if options.IsConfig { buf, err := iolimits.ReadAtMost(stream, iolimits.MaxConfigBodySize) if err != nil { return types.BlobInfo{}, perrors.Wrap(err, "reading Config file stream") @@ -137,16 +123,14 @@ func (d *Destination) PutBlob(ctx context.Context, stream io.Reader, inputInfo t return types.BlobInfo{Digest: inputInfo.Digest, Size: inputInfo.Size}, nil } -// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination +// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). // info.Digest must not be empty. -// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input. // If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may // include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be // reflected in the manifest that will be written. // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. -// May use and/or update cache. -func (d *Destination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) { +func (d *Destination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { if err := d.archive.lock(); err != nil { return false, types.BlobInfo{}, err } @@ -186,16 +170,3 @@ func (d *Destination) PutManifest(ctx context.Context, m []byte, instanceDigest return d.archive.ensureManifestItemLocked(man.LayersDescriptors, man.ConfigDescriptor.Digest, d.repoTags) } - -// PutSignatures would add the given signatures to the docker tarfile (currently not supported). -// The instanceDigest value is expected to always be nil, because this transport does not support manifest lists, so -// there can be no secondary manifests. MUST be called after PutManifest (signatures reference manifest contents). -func (d *Destination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { - if instanceDigest != nil { - return errors.New(`Manifest lists are not supported for docker tar files`) - } - if len(signatures) != 0 { - return errors.New("Storing signatures for docker tar files is not supported") - } - return nil -} diff --git a/docker/internal/tarfile/src_test.go b/docker/internal/tarfile/src_test.go index e4edbe1b0..c46eed641 100644 --- a/docker/internal/tarfile/src_test.go +++ b/docker/internal/tarfile/src_test.go @@ -28,7 +28,7 @@ func TestSourcePrepareLayerData(t *testing.T) { ctx := context.Background() writer := NewWriter(&tarfileBuffer) - dest := NewDestination(nil, writer, nil) + dest := NewDestination(nil, writer, "transport name", nil) // No layers configInfo, err := dest.PutBlob(ctx, strings.NewReader(c.config), types.BlobInfo{Size: -1}, cache, true) diff --git a/docker/tarfile/dest.go b/docker/tarfile/dest.go index 65d60c37a..454765439 100644 --- a/docker/tarfile/dest.go +++ b/docker/tarfile/dest.go @@ -26,7 +26,7 @@ func NewDestination(dest io.Writer, ref reference.NamedTagged) *Destination { func NewDestinationWithContext(sys *types.SystemContext, dest io.Writer, ref reference.NamedTagged) *Destination { archive := internal.NewWriter(dest) return &Destination{ - internal: internal.NewDestination(sys, archive, ref), + internal: internal.NewDestination(sys, archive, "[An external docker/tarfile caller]", ref), archive: archive, } } diff --git a/internal/imagedestination/impl/compat.go b/internal/imagedestination/impl/compat.go new file mode 100644 index 000000000..2cc3c7aac --- /dev/null +++ b/internal/imagedestination/impl/compat.go @@ -0,0 +1,63 @@ +package impl + +import ( + "context" + "io" + + "github.com/containers/image/v5/internal/blobinfocache" + "github.com/containers/image/v5/internal/private" + "github.com/containers/image/v5/types" +) + +// Compat implements the obsolete parts of types.ImageDestination +// for implementations of private.ImageDestination. +// See AddCompat below. +type Compat struct { + dest private.ImageDestinationInternalOnly +} + +// AddCompat initializes Compat to implement the obsolete parts of types.ImageDestination +// for implementations of private.ImageDestination. +// +// Use it like this: +// type yourDestination struct { +// impl.Compat +// … +// } +// dest := &yourDestination{…} +// dest.Compat = impl.AddCompat(dest) +// +func AddCompat(dest private.ImageDestinationInternalOnly) Compat { + return Compat{dest} +} + +// PutBlob writes contents of stream and returns data representing the result. +// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. +// inputInfo.Size is the expected length of stream, if known. +// inputInfo.MediaType describes the blob format, if known. +// May update cache. +// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available +// to any other readers for download using the supplied digest. +// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. +func (c *Compat) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) { + return c.dest.PutBlobWithOptions(ctx, stream, inputInfo, private.PutBlobOptions{ + Cache: blobinfocache.FromBlobInfoCache(cache), + IsConfig: isConfig, + }) +} + +// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination +// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). +// info.Digest must not be empty. +// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input. +// If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may +// include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be +// reflected in the manifest that will be written. +// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. +// May use and/or update cache. +func (c *Compat) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) { + return c.dest.TryReusingBlobWithOptions(ctx, info, private.TryReusingBlobOptions{ + Cache: blobinfocache.FromBlobInfoCache(cache), + CanSubstitute: canSubstitute, + }) +} diff --git a/internal/imagedestination/impl/properties.go b/internal/imagedestination/impl/properties.go new file mode 100644 index 000000000..704812e9a --- /dev/null +++ b/internal/imagedestination/impl/properties.go @@ -0,0 +1,72 @@ +package impl + +import "github.com/containers/image/v5/types" + +// Properties collects properties of an ImageDestination that are constant throughout its lifetime +// (but might differ across instances). +type Properties struct { + // SupportedManifestMIMETypes tells which manifest MIME types the destination supports. + // A empty slice or nil means any MIME type can be tried to upload. + SupportedManifestMIMETypes []string + // DesiredLayerCompression indicates the kind of compression to apply on layers + DesiredLayerCompression types.LayerCompression + // AcceptsForeignLayerURLs is false if foreign layers in manifest should be actually + // uploaded to the image destination, true otherwise. + AcceptsForeignLayerURLs bool + // MustMatchRuntimeOS is set to true if the destination can store only images targeted for the current runtime architecture and OS. + MustMatchRuntimeOS bool + // IgnoresEmbeddedDockerReference is set to true if the destination does not care about Image.EmbeddedDockerReferenceConflicts(), + // and would prefer to receive an unmodified manifest instead of one modified for the destination. + // Does not make a difference if Reference().DockerReference() is nil. + IgnoresEmbeddedDockerReference bool + // HasThreadSafePutBlob indicates that PutBlob can be executed concurrently. + HasThreadSafePutBlob bool +} + +// PropertyMethodsInitialize implements parts of private.ImageDestination corresponding to Properties. +type PropertyMethodsInitialize struct { + // We need two separate structs, PropertyMethodsInitialize and Properties, because Go prohibits fields and methods with the same name. + + vals Properties +} + +// PropertyMethods creates an PropertyMethodsInitialize for vals. +func PropertyMethods(vals Properties) PropertyMethodsInitialize { + return PropertyMethodsInitialize{ + vals: vals, + } +} + +// SupportedManifestMIMETypes tells which manifest mime types the destination supports +// If an empty slice or nil it's returned, then any mime type can be tried to upload +func (o PropertyMethodsInitialize) SupportedManifestMIMETypes() []string { + return o.vals.SupportedManifestMIMETypes +} + +// DesiredLayerCompression indicates the kind of compression to apply on layers +func (o PropertyMethodsInitialize) DesiredLayerCompression() types.LayerCompression { + return o.vals.DesiredLayerCompression +} + +// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually +// uploaded to the image destination, true otherwise. +func (o PropertyMethodsInitialize) AcceptsForeignLayerURLs() bool { + return o.vals.AcceptsForeignLayerURLs +} + +// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise. +func (o PropertyMethodsInitialize) MustMatchRuntimeOS() bool { + return o.vals.MustMatchRuntimeOS +} + +// IgnoresEmbeddedDockerReference() returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(), +// and would prefer to receive an unmodified manifest instead of one modified for the destination. +// Does not make a difference if Reference().DockerReference() is nil. +func (o PropertyMethodsInitialize) IgnoresEmbeddedDockerReference() bool { + return o.vals.IgnoresEmbeddedDockerReference +} + +// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. +func (o PropertyMethodsInitialize) HasThreadSafePutBlob() bool { + return o.vals.HasThreadSafePutBlob +} diff --git a/internal/imagedestination/stubs/put_blob_partial.go b/internal/imagedestination/stubs/put_blob_partial.go new file mode 100644 index 000000000..225ea4491 --- /dev/null +++ b/internal/imagedestination/stubs/put_blob_partial.go @@ -0,0 +1,52 @@ +package stubs + +import ( + "context" + "fmt" + + "github.com/containers/image/v5/internal/blobinfocache" + "github.com/containers/image/v5/internal/private" + "github.com/containers/image/v5/types" +) + +// NoPutBlobPartialInitialize implements parts of private.ImageDestination +// for transports that don’t support PutBlobPartial(). +// See NoPutBlobPartial() below. +type NoPutBlobPartialInitialize struct { + transportName string +} + +// NoPutBlobPartial creates a NoPutBlobPartialInitialize for ref. +func NoPutBlobPartial(ref types.ImageReference) NoPutBlobPartialInitialize { + return NoPutBlobPartialRaw(ref.Transport().Name()) +} + +// NoPutBlobPartialRaw is the same thing as NoPutBlobPartial, but it can be used +// in situations where no ImageReference is available. +func NoPutBlobPartialRaw(transportName string) NoPutBlobPartialInitialize { + return NoPutBlobPartialInitialize{ + transportName: transportName, + } +} + +// SupportsPutBlobPartial returns true if PutBlobPartial is supported. +func (stub NoPutBlobPartialInitialize) SupportsPutBlobPartial() bool { + return false +} + +// PutBlobPartial attempts to create a blob using the data that is already present +// at the destination. chunkAccessor is accessed in a non-sequential way to retrieve the missing chunks. +// It is available only if SupportsPutBlobPartial(). +// Even if SupportsPutBlobPartial() returns true, the call can fail, in which case the caller +// should fall back to PutBlobWithOptions. +func (stub NoPutBlobPartialInitialize) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, cache blobinfocache.BlobInfoCache2) (types.BlobInfo, error) { + return types.BlobInfo{}, fmt.Errorf("internal error: PutBlobPartial is not supported by the %q transport", stub.transportName) +} + +// ImplementsPutBlobPartial implements SupportsPutBlobPartial() that returns true. +type ImplementsPutBlobPartial struct{} + +// SupportsPutBlobPartial returns true if PutBlobPartial is supported. +func (stub ImplementsPutBlobPartial) SupportsPutBlobPartial() bool { + return true +} diff --git a/internal/imagedestination/stubs/signatures.go b/internal/imagedestination/stubs/signatures.go new file mode 100644 index 000000000..2e3c2eadb --- /dev/null +++ b/internal/imagedestination/stubs/signatures.go @@ -0,0 +1,49 @@ +package stubs + +import ( + "context" + "errors" + + "github.com/opencontainers/go-digest" +) + +// NoSignaturesInitialize implements parts of private.ImageDestination +// for transports that don’t support storing signatures. +// See NoSignatures() below. +type NoSignaturesInitialize struct { + message string +} + +// NoSignatures creates a NoSignaturesInitialize, failing with message. +func NoSignatures(message string) NoSignaturesInitialize { + return NoSignaturesInitialize{ + message: message, + } +} + +// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures. +// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil. +func (stub NoSignaturesInitialize) SupportsSignatures(ctx context.Context) error { + return errors.New(stub.message) +} + +// PutSignatures writes a set of signatures to the destination. +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for +// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. +// MUST be called after PutManifest (signatures may reference manifest contents). +func (stub NoSignaturesInitialize) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { + if len(signatures) != 0 { + return errors.New(stub.message) + } + return nil +} + +// SupportsSignatures implements SupportsSignatures() that returns nil. +// Note that it might be even more useful to return a value dynamically detected based on +type AlwaysSupportsSignatures struct{} + +// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures. +// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil. +func (stub AlwaysSupportsSignatures) SupportsSignatures(ctx context.Context) error { + return nil +} diff --git a/internal/imagedestination/stubs/stubs.go b/internal/imagedestination/stubs/stubs.go new file mode 100644 index 000000000..e81eec896 --- /dev/null +++ b/internal/imagedestination/stubs/stubs.go @@ -0,0 +1,25 @@ +// Package stubs contains trivial stubs for parts of private.ImageDestination. +// It can be used from internal/wrapper, so it should not drag in any extra dependencies. +// Compare with imagedestination/impl, which might require non-trivial implementation work. +// +// There are two kinds of stubs: +// - Pure stubs, like ImplementsPutBlobPartial. Those can just be included in an imageDestination +// implementation: +// +// type yourDestination struct { +// stubs.ImplementsPutBlobPartial +// … +// } +// - Stubs with a constructor, like NoPutBlobPartialInitialize. The Initialize marker +// means that a constructor must be called: +// type yourDestination struct { +// stubs.NoPutBlobPartialInitialize +// … +// } +// +// dest := &yourDestination{ +// … +// NoPutBlobPartialInitialize: stubs.NoPutBlobPartial(ref), +// } +// +package stubs diff --git a/internal/imagedestination/wrapper.go b/internal/imagedestination/wrapper.go index 82734a6cd..429f546ad 100644 --- a/internal/imagedestination/wrapper.go +++ b/internal/imagedestination/wrapper.go @@ -2,13 +2,21 @@ package imagedestination import ( "context" - "fmt" "io" + "github.com/containers/image/v5/internal/imagedestination/stubs" "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/types" ) +// wrapped provides the private.ImageDestination operations +// for a destination that only implements types.ImageDestination +type wrapped struct { + stubs.NoPutBlobPartialInitialize + + types.ImageDestination +} + // FromPublic(dest) returns an object that provides the private.ImageDestination API // // Eventually, we might want to expose this function, and methods of the returned object, @@ -23,18 +31,11 @@ func FromPublic(dest types.ImageDestination) private.ImageDestination { if dest2, ok := dest.(private.ImageDestination); ok { return dest2 } - return &wrapped{ImageDestination: dest} -} + return &wrapped{ + NoPutBlobPartialInitialize: stubs.NoPutBlobPartial(dest.Reference()), -// wrapped provides the private.ImageDestination operations -// for a destination that only implements types.ImageDestination -type wrapped struct { - types.ImageDestination -} - -// SupportsPutBlobPartial returns true if PutBlobPartial is supported. -func (w *wrapped) SupportsPutBlobPartial() bool { - return false + ImageDestination: dest, + } } // PutBlobWithOptions writes contents of stream and returns data representing the result. @@ -48,15 +49,6 @@ func (w *wrapped) PutBlobWithOptions(ctx context.Context, stream io.Reader, inpu return w.PutBlob(ctx, stream, inputInfo, options.Cache, options.IsConfig) } -// PutBlobPartial attempts to create a blob using the data that is already present -// at the destination. chunkAccessor is accessed in a non-sequential way to retrieve the missing chunks. -// It is available only if SupportsPutBlobPartial(). -// Even if SupportsPutBlobPartial() returns true, the call can fail, in which case the caller -// should fall back to PutBlobWithOptions. -func (w *wrapped) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, cache types.BlobInfoCache) (types.BlobInfo, error) { - return types.BlobInfo{}, fmt.Errorf("internal error: PutBlobPartial is not supported by the %q transport", w.Reference().Transport().Name()) -} - // TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). // info.Digest must not be empty. diff --git a/internal/private/private.go b/internal/private/private.go index 65788651f..efc44d9c7 100644 --- a/internal/private/private.go +++ b/internal/private/private.go @@ -5,6 +5,7 @@ import ( "io" "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/internal/blobinfocache" "github.com/containers/image/v5/types" ) @@ -18,11 +19,9 @@ type ImageSource interface { BlobChunkAccessor } -// ImageDestination is an internal extension to the types.ImageDestination -// interface. -type ImageDestination interface { - types.ImageDestination - +// ImageDestinationInternalOnly is the part of private.ImageDestination that is not +// a part of types.ImageDestination. +type ImageDestinationInternalOnly interface { // SupportsPutBlobPartial returns true if PutBlobPartial is supported. SupportsPutBlobPartial() bool @@ -40,7 +39,7 @@ type ImageDestination interface { // It is available only if SupportsPutBlobPartial(). // Even if SupportsPutBlobPartial() returns true, the call can fail, in which case the caller // should fall back to PutBlobWithOptions. - PutBlobPartial(ctx context.Context, chunkAccessor BlobChunkAccessor, srcInfo types.BlobInfo, cache types.BlobInfoCache) (types.BlobInfo, error) + PutBlobPartial(ctx context.Context, chunkAccessor BlobChunkAccessor, srcInfo types.BlobInfo, cache blobinfocache.BlobInfoCache2) (types.BlobInfo, error) // TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). @@ -52,13 +51,23 @@ type ImageDestination interface { TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options TryReusingBlobOptions) (bool, types.BlobInfo, error) } +// ImageDestination is an internal extension to the types.ImageDestination +// interface. +type ImageDestination interface { + types.ImageDestination + ImageDestinationInternalOnly +} + // PutBlobOptions are used in PutBlobWithOptions. type PutBlobOptions struct { - Cache types.BlobInfoCache // Cache to optionally update with the uploaded bloblook up blob infos. - IsConfig bool // True if the blob is a config + Cache blobinfocache.BlobInfoCache2 // Cache to optionally update with the uploaded bloblook up blob infos. + IsConfig bool // True if the blob is a config // The following fields are new to internal/private. Users of internal/private MUST fill them in, // but they also must expect that they will be ignored by types.ImageDestination transports. + // Transports, OTOH, MUST support these fields being zero-valued for types.ImageDestination callers + // if they use internal/imagedestination/impl.Compat; + // in that case, they will all be consistently zero-valued. EmptyLayer bool // True if the blob is an "empty"/"throwaway" layer, and may not necessarily be physically represented. LayerIndex *int // If the blob is a layer, a zero-based index of the layer within the image; nil otherwise. @@ -66,13 +75,16 @@ type PutBlobOptions struct { // TryReusingBlobOptions are used in TryReusingBlobWithOptions. type TryReusingBlobOptions struct { - Cache types.BlobInfoCache // Cache to use and/or update. + Cache blobinfocache.BlobInfoCache2 // Cache to use and/or update. // If true, it is allowed to use an equivalent of the desired blob; // in that case the returned info may not match the input. CanSubstitute bool // The following fields are new to internal/private. Users of internal/private MUST fill them in, // but they also must expect that they will be ignored by types.ImageDestination transports. + // Transports, OTOH, MUST support these fields being zero-valued for types.ImageDestination callers + // if they use internal/imagedestination/impl.Compat; + // in that case, they will all be consistently zero-valued. EmptyLayer bool // True if the blob is an "empty"/"throwaway" layer, and may not necessarily be physically represented. LayerIndex *int // If the blob is a layer, a zero-based index of the layer within the image; nil otherwise. diff --git a/oci/archive/oci_dest.go b/oci/archive/oci_dest.go index a0eaac767..3ba9f6d3b 100644 --- a/oci/archive/oci_dest.go +++ b/oci/archive/oci_dest.go @@ -5,6 +5,10 @@ import ( "io" "os" + "github.com/containers/image/v5/internal/blobinfocache" + "github.com/containers/image/v5/internal/imagedestination" + "github.com/containers/image/v5/internal/imagedestination/impl" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/types" "github.com/containers/storage/pkg/archive" digest "github.com/opencontainers/go-digest" @@ -13,13 +17,15 @@ import ( ) type ociArchiveImageDestination struct { + impl.Compat + ref ociArchiveReference - unpackedDest types.ImageDestination + unpackedDest private.ImageDestination tempDirRef tempDirOCIRef } // newImageDestination returns an ImageDestination for writing to an existing directory. -func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageDestination, error) { +func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (private.ImageDestination, error) { tempDirRef, err := createOCIRef(sys, ref.image) if err != nil { return nil, perrors.Wrapf(err, "creating oci reference") @@ -31,9 +37,13 @@ func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociA } return nil, err } - return &ociArchiveImageDestination{ref: ref, - unpackedDest: unpackedDest, - tempDirRef: tempDirRef}, nil + d := &ociArchiveImageDestination{ + ref: ref, + unpackedDest: imagedestination.FromPublic(unpackedDest), + tempDirRef: tempDirRef, + } + d.Compat = impl.AddCompat(d) + return d, nil } // Reference returns the reference used to set up this destination. @@ -87,29 +97,40 @@ func (d *ociArchiveImageDestination) HasThreadSafePutBlob() bool { return false } -// PutBlob writes contents of stream and returns data representing the result. +// SupportsPutBlobPartial returns true if PutBlobPartial is supported. +func (d *ociArchiveImageDestination) SupportsPutBlobPartial() bool { + return d.unpackedDest.SupportsPutBlobPartial() +} + +// PutBlobWithOptions writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. // inputInfo.Size is the expected length of stream, if known. // inputInfo.MediaType describes the blob format, if known. -// May update cache. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (d *ociArchiveImageDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) { - return d.unpackedDest.PutBlob(ctx, stream, inputInfo, cache, isConfig) +func (d *ociArchiveImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options private.PutBlobOptions) (types.BlobInfo, error) { + return d.unpackedDest.PutBlobWithOptions(ctx, stream, inputInfo, options) +} + +// PutBlobPartial attempts to create a blob using the data that is already present +// at the destination. chunkAccessor is accessed in a non-sequential way to retrieve the missing chunks. +// It is available only if SupportsPutBlobPartial(). +// Even if SupportsPutBlobPartial() returns true, the call can fail, in which case the caller +// should fall back to PutBlobWithOptions. +func (d *ociArchiveImageDestination) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, cache blobinfocache.BlobInfoCache2) (types.BlobInfo, error) { + return d.unpackedDest.PutBlobPartial(ctx, chunkAccessor, srcInfo, cache) } -// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination +// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). // info.Digest must not be empty. -// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input. // If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may // include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be // reflected in the manifest that will be written. // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. -// May use and/or update cache. -func (d *ociArchiveImageDestination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) { - return d.unpackedDest.TryReusingBlob(ctx, info, cache, canSubstitute) +func (d *ociArchiveImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { + return d.unpackedDest.TryReusingBlobWithOptions(ctx, info, options) } // PutManifest writes the manifest to the destination. diff --git a/oci/archive/oci_dest_test.go b/oci/archive/oci_dest_test.go new file mode 100644 index 000000000..a67112cf3 --- /dev/null +++ b/oci/archive/oci_dest_test.go @@ -0,0 +1,5 @@ +package archive + +import "github.com/containers/image/v5/internal/private" + +var _ private.ImageDestination = (*ociArchiveImageDestination)(nil) diff --git a/oci/layout/oci_dest.go b/oci/layout/oci_dest.go index 4b23b70b8..4face7213 100644 --- a/oci/layout/oci_dest.go +++ b/oci/layout/oci_dest.go @@ -10,6 +10,9 @@ import ( "path/filepath" "runtime" + "github.com/containers/image/v5/internal/imagedestination/impl" + "github.com/containers/image/v5/internal/imagedestination/stubs" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/internal/putblobdigest" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/types" @@ -19,14 +22,18 @@ import ( ) type ociImageDestination struct { - ref ociReference - index imgspecv1.Index - sharedBlobDir string - acceptUncompressedLayers bool + impl.Compat + impl.PropertyMethodsInitialize + stubs.NoPutBlobPartialInitialize + stubs.NoSignaturesInitialize + + ref ociReference + index imgspecv1.Index + sharedBlobDir string } // newImageDestination returns an ImageDestination for writing to an existing directory. -func newImageDestination(sys *types.SystemContext, ref ociReference) (types.ImageDestination, error) { +func newImageDestination(sys *types.SystemContext, ref ociReference) (private.ImageDestination, error) { var index *imgspecv1.Index if indexExists(ref) { var err error @@ -43,10 +50,32 @@ func newImageDestination(sys *types.SystemContext, ref ociReference) (types.Imag } } - d := &ociImageDestination{ref: ref, index: *index} + desiredLayerCompression := types.Compress + if sys != nil && sys.OCIAcceptUncompressedLayers { + desiredLayerCompression = types.PreserveOriginal + } + + d := &ociImageDestination{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + SupportedManifestMIMETypes: []string{ + imgspecv1.MediaTypeImageManifest, + imgspecv1.MediaTypeImageIndex, + }, + DesiredLayerCompression: desiredLayerCompression, + AcceptsForeignLayerURLs: true, + MustMatchRuntimeOS: false, + IgnoresEmbeddedDockerReference: false, // N/A, DockerReference() returns nil. + HasThreadSafePutBlob: true, + }), + NoPutBlobPartialInitialize: stubs.NoPutBlobPartial(ref), + NoSignaturesInitialize: stubs.NoSignatures("Pushing signatures for OCI images is not supported"), + + ref: ref, + index: *index, + } + d.Compat = impl.AddCompat(d) if sys != nil { d.sharedBlobDir = sys.OCISharedBlobDirPath - d.acceptUncompressedLayers = sys.OCIAcceptUncompressedLayers } if err := ensureDirectoryExists(d.ref.dir); err != nil { @@ -72,58 +101,14 @@ func (d *ociImageDestination) Close() error { return nil } -func (d *ociImageDestination) SupportedManifestMIMETypes() []string { - return []string{ - imgspecv1.MediaTypeImageManifest, - imgspecv1.MediaTypeImageIndex, - } -} - -// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures. -// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil. -func (d *ociImageDestination) SupportsSignatures(ctx context.Context) error { - return errors.New("Pushing signatures for OCI images is not supported") -} - -func (d *ociImageDestination) DesiredLayerCompression() types.LayerCompression { - if d.acceptUncompressedLayers { - return types.PreserveOriginal - } - return types.Compress -} - -// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually -// uploaded to the image destination, true otherwise. -func (d *ociImageDestination) AcceptsForeignLayerURLs() bool { - return true -} - -// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise. -func (d *ociImageDestination) MustMatchRuntimeOS() bool { - return false -} - -// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(), -// and would prefer to receive an unmodified manifest instead of one modified for the destination. -// Does not make a difference if Reference().DockerReference() is nil. -func (d *ociImageDestination) IgnoresEmbeddedDockerReference() bool { - return false // N/A, DockerReference() returns nil. -} - -// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. -func (d *ociImageDestination) HasThreadSafePutBlob() bool { - return true -} - -// PutBlob writes contents of stream and returns data representing the result. +// PutBlobWithOptions writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. // inputInfo.Size is the expected length of stream, if known. // inputInfo.MediaType describes the blob format, if known. -// May update cache. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (d *ociImageDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) { +func (d *ociImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options private.PutBlobOptions) (types.BlobInfo, error) { blobFile, err := os.CreateTemp(d.ref.dir, "oci-put-blob") if err != nil { return types.BlobInfo{}, err @@ -181,16 +166,14 @@ func (d *ociImageDestination) PutBlob(ctx context.Context, stream io.Reader, inp return types.BlobInfo{Digest: blobDigest, Size: size}, nil } -// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination +// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). // info.Digest must not be empty. -// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input. // If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may // include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be // reflected in the manifest that will be written. // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. -// May use and/or update cache. -func (d *ociImageDestination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) { +func (d *ociImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { if info.Digest == "" { return false, types.BlobInfo{}, errors.New("Can not check for a blob with unknown digest") } @@ -291,16 +274,6 @@ func (d *ociImageDestination) addManifest(desc *imgspecv1.Descriptor) { d.index.Manifests = append(d.index.Manifests, *desc) } -// PutSignatures would add the given signatures to the oci layout (currently not supported). -// If instanceDigest is not nil, it contains a digest of the specific manifest instance to write or overwrite the signatures for -// (when the primary manifest is a manifest list); this should always be nil if the primary manifest is not a manifest list. -func (d *ociImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { - if len(signatures) != 0 { - return errors.New("Pushing signatures for OCI images is not supported") - } - return nil -} - // Commit marks the process of storing the image as successful and asks for the image to be persisted. // unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list // if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the diff --git a/oci/layout/oci_dest_test.go b/oci/layout/oci_dest_test.go index bdeef581a..33cf8516b 100644 --- a/oci/layout/oci_dest_test.go +++ b/oci/layout/oci_dest_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "testing" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/pkg/blobinfocache/memory" "github.com/containers/image/v5/types" digest "github.com/opencontainers/go-digest" @@ -17,6 +18,8 @@ import ( "github.com/stretchr/testify/require" ) +var _ private.ImageDestination = (*ociImageDestination)(nil) + // readerFromFunc allows implementing Reader by any function, e.g. a closure. type readerFromFunc func([]byte) (int, error) diff --git a/openshift/openshift.go b/openshift/openshift.go index c67b4d1e0..6147b9c0d 100644 --- a/openshift/openshift.go +++ b/openshift/openshift.go @@ -14,7 +14,12 @@ import ( "github.com/containers/image/v5/docker" "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/internal/blobinfocache" + "github.com/containers/image/v5/internal/imagedestination" + "github.com/containers/image/v5/internal/imagedestination/impl" + "github.com/containers/image/v5/internal/imagedestination/stubs" "github.com/containers/image/v5/internal/iolimits" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/types" "github.com/containers/image/v5/version" @@ -315,14 +320,17 @@ func (s *openshiftImageSource) ensureImageIsResolved(ctx context.Context) error } type openshiftImageDestination struct { + impl.Compat + stubs.AlwaysSupportsSignatures + client *openshiftClient - docker types.ImageDestination // The docker/distribution API endpoint + docker private.ImageDestination // The docker/distribution API endpoint // State imageStreamImageName string // "" if not yet known } // newImageDestination creates a new ImageDestination for the specified reference. -func newImageDestination(ctx context.Context, sys *types.SystemContext, ref openshiftReference) (types.ImageDestination, error) { +func newImageDestination(ctx context.Context, sys *types.SystemContext, ref openshiftReference) (private.ImageDestination, error) { client, err := newOpenshiftClient(ref) if err != nil { return nil, err @@ -341,10 +349,12 @@ func newImageDestination(ctx context.Context, sys *types.SystemContext, ref open return nil, err } - return &openshiftImageDestination{ + d := &openshiftImageDestination{ client: client, - docker: docker, - }, nil + docker: imagedestination.FromPublic(docker), + } + d.Compat = impl.AddCompat(d) + return d, nil } // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, @@ -362,12 +372,6 @@ func (d *openshiftImageDestination) SupportedManifestMIMETypes() []string { return d.docker.SupportedManifestMIMETypes() } -// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures. -// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil. -func (d *openshiftImageDestination) SupportsSignatures(ctx context.Context) error { - return nil -} - func (d *openshiftImageDestination) DesiredLayerCompression() types.LayerCompression { return types.Compress } @@ -395,28 +399,40 @@ func (d *openshiftImageDestination) HasThreadSafePutBlob() bool { return false } -// PutBlob writes contents of stream and returns data representing the result (with all data filled in). +// SupportsPutBlobPartial returns true if PutBlobPartial is supported. +func (d *openshiftImageDestination) SupportsPutBlobPartial() bool { + return d.docker.SupportsPutBlobPartial() +} + +// PutBlobWithOptions writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. // inputInfo.Size is the expected length of stream, if known. -// May update cache. +// inputInfo.MediaType describes the blob format, if known. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (d *openshiftImageDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) { - return d.docker.PutBlob(ctx, stream, inputInfo, cache, isConfig) +func (d *openshiftImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options private.PutBlobOptions) (types.BlobInfo, error) { + return d.docker.PutBlobWithOptions(ctx, stream, inputInfo, options) +} + +// PutBlobPartial attempts to create a blob using the data that is already present +// at the destination. chunkAccessor is accessed in a non-sequential way to retrieve the missing chunks. +// It is available only if SupportsPutBlobPartial(). +// Even if SupportsPutBlobPartial() returns true, the call can fail, in which case the caller +// should fall back to PutBlobWithOptions. +func (d *openshiftImageDestination) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, cache blobinfocache.BlobInfoCache2) (types.BlobInfo, error) { + return d.docker.PutBlobPartial(ctx, chunkAccessor, srcInfo, cache) } -// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination +// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). // info.Digest must not be empty. -// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input. // If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may // include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be // reflected in the manifest that will be written. // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. -// May use and/or update cache. -func (d *openshiftImageDestination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) { - return d.docker.TryReusingBlob(ctx, info, cache, canSubstitute) +func (d *openshiftImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { + return d.docker.TryReusingBlobWithOptions(ctx, info, options) } // PutManifest writes manifest to the destination. diff --git a/openshift/openshift_test.go b/openshift/openshift_test.go new file mode 100644 index 000000000..795aa7111 --- /dev/null +++ b/openshift/openshift_test.go @@ -0,0 +1,5 @@ +package openshift + +import "github.com/containers/image/v5/internal/private" + +var _ private.ImageDestination = (*openshiftImageDestination)(nil) diff --git a/ostree/ostree_dest.go b/ostree/ostree_dest.go index ec2aa2744..ae8a398bd 100644 --- a/ostree/ostree_dest.go +++ b/ostree/ostree_dest.go @@ -21,6 +21,9 @@ import ( "time" "unsafe" + "github.com/containers/image/v5/internal/imagedestination/impl" + "github.com/containers/image/v5/internal/imagedestination/stubs" + "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/internal/putblobdigest" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/types" @@ -66,6 +69,11 @@ type manifestSchema struct { } type ostreeImageDestination struct { + compat impl.Compat + impl.PropertyMethodsInitialize + stubs.NoPutBlobPartialInitialize + stubs.AlwaysSupportsSignatures + ref ostreeReference manifest string schema manifestSchema @@ -77,12 +85,33 @@ type ostreeImageDestination struct { } // newImageDestination returns an ImageDestination for writing to an existing ostree. -func newImageDestination(ref ostreeReference, tmpDirPath string) (types.ImageDestination, error) { +func newImageDestination(ref ostreeReference, tmpDirPath string) (private.ImageDestination, error) { tmpDirPath = filepath.Join(tmpDirPath, ref.branchName) if err := ensureDirectoryExists(tmpDirPath); err != nil { return nil, err } - return &ostreeImageDestination{ref, "", manifestSchema{}, tmpDirPath, map[string]*blobToImport{}, "", 0, nil}, nil + d := &ostreeImageDestination{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + SupportedManifestMIMETypes: []string{manifest.DockerV2Schema2MediaType}, + DesiredLayerCompression: types.PreserveOriginal, + AcceptsForeignLayerURLs: false, + MustMatchRuntimeOS: true, + IgnoresEmbeddedDockerReference: false, // N/A, DockerReference() returns nil. + HasThreadSafePutBlob: false, + }), + NoPutBlobPartialInitialize: stubs.NoPutBlobPartial(ref), + + ref: ref, + manifest: "", + schema: manifestSchema{}, + tmpDirPath: tmpDirPath, + blobs: map[string]*blobToImport{}, + digest: "", + signaturesLen: 0, + repo: nil, + } + d.Compat = impl.AddCompat(d) + return d, nil } // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, @@ -99,55 +128,14 @@ func (d *ostreeImageDestination) Close() error { return os.RemoveAll(d.tmpDirPath) } -func (d *ostreeImageDestination) SupportedManifestMIMETypes() []string { - return []string{ - manifest.DockerV2Schema2MediaType, - } -} - -// SupportsSignatures returns an error (to be displayed to the user) if the destination certainly can't store signatures. -// Note: It is still possible for PutSignatures to fail if SupportsSignatures returns nil. -func (d *ostreeImageDestination) SupportsSignatures(ctx context.Context) error { - return nil -} - -// ShouldCompressLayers returns true iff it is desirable to compress layer blobs written to this destination. -func (d *ostreeImageDestination) DesiredLayerCompression() types.LayerCompression { - return types.PreserveOriginal -} - -// AcceptsForeignLayerURLs returns false iff foreign layers in manifest should be actually -// uploaded to the image destination, true otherwise. -func (d *ostreeImageDestination) AcceptsForeignLayerURLs() bool { - return false -} - -// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise. -func (d *ostreeImageDestination) MustMatchRuntimeOS() bool { - return true -} - -// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(), -// and would prefer to receive an unmodified manifest instead of one modified for the destination. -// Does not make a difference if Reference().DockerReference() is nil. -func (d *ostreeImageDestination) IgnoresEmbeddedDockerReference() bool { - return false // N/A, DockerReference() returns nil. -} - -// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. -func (d *ostreeImageDestination) HasThreadSafePutBlob() bool { - return false -} - -// PutBlob writes contents of stream and returns data representing the result. +// PutBlobWithOptions writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. // inputInfo.Size is the expected length of stream, if known. // inputInfo.MediaType describes the blob format, if known. -// May update cache. // WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (d *ostreeImageDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) { +func (d *ostreeImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options private.PutBlobOptions) (types.BlobInfo, error) { tmpDir, err := os.MkdirTemp(d.tmpDirPath, "blob") if err != nil { return types.BlobInfo{}, err @@ -339,16 +327,14 @@ func (d *ostreeImageDestination) importConfig(repo *otbuiltin.Repo, blob *blobTo return d.ostreeCommit(repo, ostreeBranch, destinationPath, []string{fmt.Sprintf("docker.size=%d", blob.Size)}) } -// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination +// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). // info.Digest must not be empty. -// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input. // If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may // include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be // reflected in the manifest that will be written. // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. -// May use and/or update cache. -func (d *ostreeImageDestination) TryReusingBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) { +func (d *ostreeImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { if d.repo == nil { repo, err := openRepo(d.ref.repo) if err != nil { diff --git a/ostree/ostree_dest_test.go b/ostree/ostree_dest_test.go new file mode 100644 index 000000000..bca3d7ce0 --- /dev/null +++ b/ostree/ostree_dest_test.go @@ -0,0 +1,6 @@ +//go:build containers_image_ostree +// +build containers_image_ostree + +package ostree + +var _ private.ImageDestination = (*ostreeImageDestination)(nil) diff --git a/storage/storage_image.go b/storage/storage_image.go index 23b37b559..e81f0ea9c 100644 --- a/storage/storage_image.go +++ b/storage/storage_image.go @@ -16,7 +16,10 @@ import ( "sync/atomic" "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/internal/blobinfocache" "github.com/containers/image/v5/internal/image" + "github.com/containers/image/v5/internal/imagedestination/impl" + "github.com/containers/image/v5/internal/imagedestination/stubs" "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/internal/putblobdigest" "github.com/containers/image/v5/internal/tmpdir" @@ -60,6 +63,11 @@ type storageImageSource struct { } type storageImageDestination struct { + impl.Compat + impl.PropertyMethodsInitialize + stubs.ImplementsPutBlobPartial + stubs.AlwaysSupportsSignatures + imageRef storageReference directory string // Temporary directory where we store blobs until Commit() time nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs @@ -398,7 +406,24 @@ func newImageDestination(sys *types.SystemContext, imageRef storageReference) (* if err != nil { return nil, perrors.Wrapf(err, "creating a temporary directory") } - image := &storageImageDestination{ + dest := &storageImageDestination{ + PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{ + SupportedManifestMIMETypes: []string{ + imgspecv1.MediaTypeImageManifest, + manifest.DockerV2Schema2MediaType, + manifest.DockerV2Schema1SignedMediaType, + manifest.DockerV2Schema1MediaType, + }, + // We ultimately have to decompress layers to populate trees on disk + // and need to explicitly ask for it here, so that the layers' MIME + // types can be set accordingly. + DesiredLayerCompression: types.PreserveOriginal, + AcceptsForeignLayerURLs: false, + MustMatchRuntimeOS: true, + IgnoresEmbeddedDockerReference: true, // Yes, we want the unmodified manifest + HasThreadSafePutBlob: true, + }), + imageRef: imageRef, directory: directory, signatureses: make(map[digest.Digest][]byte), @@ -412,7 +437,8 @@ func newImageDestination(sys *types.SystemContext, imageRef storageReference) (* indexToPulledLayerInfo: make(map[int]*manifest.LayerInfo), diffOutputs: make(map[digest.Digest]*graphdriver.DriverWithDifferOutput), } - return image, nil + dest.Compat = impl.AddCompat(dest) + return dest, nil } // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, @@ -434,22 +460,10 @@ func (s *storageImageDestination) Close() error { return os.RemoveAll(s.directory) } -func (s *storageImageDestination) DesiredLayerCompression() types.LayerCompression { - // We ultimately have to decompress layers to populate trees on disk - // and need to explicitly ask for it here, so that the layers' MIME - // types can be set accordingly. - return types.PreserveOriginal -} - func (s *storageImageDestination) computeNextBlobCacheFile() string { return filepath.Join(s.directory, fmt.Sprintf("%d", atomic.AddInt32(&s.nextTempFileID, 1))) } -// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. -func (s *storageImageDestination) HasThreadSafePutBlob() bool { - return true -} - // PutBlobWithOptions writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. // inputInfo.Size is the expected length of stream, if known. @@ -470,21 +484,6 @@ func (s *storageImageDestination) PutBlobWithOptions(ctx context.Context, stream return info, s.queueOrCommit(ctx, info, *options.LayerIndex, options.EmptyLayer) } -// PutBlob writes contents of stream and returns data representing the result. -// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents. -// inputInfo.Size is the expected length of stream, if known. -// inputInfo.MediaType describes the blob format, if known. -// May update cache. -// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available -// to any other readers for download using the supplied digest. -// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. -func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) { - return s.PutBlobWithOptions(ctx, stream, blobinfo, private.PutBlobOptions{ - Cache: cache, - IsConfig: isConfig, - }) -} - // putBlobToPendingFile implements ImageDestination.PutBlobWithOptions, storing stream into an on-disk file. // The caller must arrange the blob to be eventually committed using s.commitLayer(). func (s *storageImageDestination) putBlobToPendingFile(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, options *private.PutBlobOptions) (types.BlobInfo, error) { @@ -579,7 +578,7 @@ func (f *zstdFetcher) GetBlobAt(chunks []chunked.ImageSourceChunk) (chan io.Read // It is available only if SupportsPutBlobPartial(). // Even if SupportsPutBlobPartial() returns true, the call can fail, in which case the caller // should fall back to PutBlobWithOptions. -func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, cache types.BlobInfoCache) (types.BlobInfo, error) { +func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAccessor private.BlobChunkAccessor, srcInfo types.BlobInfo, cache blobinfocache.BlobInfoCache2) (types.BlobInfo, error) { fetcher := zstdFetcher{ chunkAccessor: chunkAccessor, ctx: ctx, @@ -624,22 +623,6 @@ func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context, return reused, info, s.queueOrCommit(ctx, info, *options.LayerIndex, options.EmptyLayer) } -// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination -// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). -// info.Digest must not be empty. -// If canSubstitute, TryReusingBlob can use an equivalent equivalent of the desired blob; in that case the returned info may not match the input. -// If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may -// include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be -// reflected in the manifest that will be written. -// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. -// May use and/or update cache. -func (s *storageImageDestination) TryReusingBlob(ctx context.Context, blobinfo types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) { - return s.TryReusingBlobWithOptions(ctx, blobinfo, private.TryReusingBlobOptions{ - Cache: cache, - CanSubstitute: canSubstitute, - }) -} - // tryReusingBlobAsPending implements TryReusingBlobWithOptions, filling s.blobDiffIDs and other metadata. // The caller must arrange the blob to be eventually committed using s.commitLayer(). func (s *storageImageDestination) tryReusingBlobAsPending(ctx context.Context, blobinfo types.BlobInfo, options *private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { @@ -1209,17 +1192,6 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t return nil } -var manifestMIMETypes = []string{ - imgspecv1.MediaTypeImageManifest, - manifest.DockerV2Schema2MediaType, - manifest.DockerV2Schema1SignedMediaType, - manifest.DockerV2Schema1MediaType, -} - -func (s *storageImageDestination) SupportedManifestMIMETypes() []string { - return manifestMIMETypes -} - // PutManifest writes the manifest to the destination. func (s *storageImageDestination) PutManifest(ctx context.Context, manifestBlob []byte, instanceDigest *digest.Digest) error { digest, err := manifest.Digest(manifestBlob) @@ -1233,35 +1205,6 @@ func (s *storageImageDestination) PutManifest(ctx context.Context, manifestBlob return nil } -// SupportsSignatures returns an error if we can't expect GetSignatures() to return data that was -// previously supplied to PutSignatures(). -func (s *storageImageDestination) SupportsSignatures(ctx context.Context) error { - return nil -} - -// AcceptsForeignLayerURLs returns false iff foreign layers in the manifest should actually be -// uploaded to the image destination, true otherwise. -func (s *storageImageDestination) AcceptsForeignLayerURLs() bool { - return false -} - -// MustMatchRuntimeOS returns true iff the destination can store only images targeted for the current runtime architecture and OS. False otherwise. -func (s *storageImageDestination) MustMatchRuntimeOS() bool { - return true -} - -// IgnoresEmbeddedDockerReference returns true iff the destination does not care about Image.EmbeddedDockerReferenceConflicts(), -// and would prefer to receive an unmodified manifest instead of one modified for the destination. -// Does not make a difference if Reference().DockerReference() is nil. -func (s *storageImageDestination) IgnoresEmbeddedDockerReference() bool { - return true // Yes, we want the unmodified manifest -} - -// SupportsPutBlobPartial returns true if PutBlobPartial is supported. -func (s *storageImageDestination) SupportsPutBlobPartial() bool { - return true -} - // PutSignatures records the image's signatures for committing as a single data blob. func (s *storageImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error { sizes := []int{}