From a8b2234b3228fa1fe5ea7dc85f6dcebbcc970ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= Date: Tue, 14 Jun 2022 22:39:19 +0200 Subject: [PATCH] XX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split copy pipeline up Signed-off-by: Miloslav Trmač --- copy/compression.go | 27 +++++++++++++++------------ internal/image/docker_schema1.go | 16 ++++++++++++++++ internal/image/docker_schema2.go | 15 +++++++++++++++ internal/image/manifest.go | 8 ++++++++ internal/image/oci.go | 15 +++++++++++++++ manifest/docker_schema2.go | 16 ++++++++++++++++ manifest/oci.go | 22 ++++++++++++++++++++++ 7 files changed, 107 insertions(+), 12 deletions(-) diff --git a/copy/compression.go b/copy/compression.go index f6132dbe96..8302799d5c 100644 --- a/copy/compression.go +++ b/copy/compression.go @@ -104,16 +104,17 @@ func (ic *imageCopier) bpcPreserveEncrypted(stream *sourceStream, _ bpDetectComp // bpcCompressUncompressed checks if we should be compressing an uncompressed input, and returns a *bpCompressionStepData if so. func (ic *imageCopier) bpcCompressUncompressed(stream *sourceStream, detected bpDetectCompressionStepData) (*bpCompressionStepData, error) { - if ic.c.dest.DesiredLayerCompression() == types.Compress && !detected.isCompressed { + var chosenFormat *compressiontypes.Algorithm + if ic.c.compressionFormat != nil { + chosenFormat = ic.c.compressionFormat + } else { + chosenFormat = defaultCompressionFormat + } + if ic.c.dest.DesiredLayerCompression() == types.Compress && !detected.isCompressed && + ic.src.CanCompressLayer(stream.info.MediaType, chosenFormat) { logrus.Debugf("Compressing blob on the fly") - var uploadedAlgorithm *compressiontypes.Algorithm - if ic.c.compressionFormat != nil { - uploadedAlgorithm = ic.c.compressionFormat - } else { - uploadedAlgorithm = defaultCompressionFormat - } - reader, annotations := ic.c.compressedStream(stream.reader, *uploadedAlgorithm) + reader, annotations := ic.c.compressedStream(stream.reader, *chosenFormat) // Note: reader must be closed on all return paths. stream.reader = reader stream.info = types.BlobInfo{ // FIXME? Should we preserve more data in src.info? @@ -122,10 +123,10 @@ func (ic *imageCopier) bpcCompressUncompressed(stream *sourceStream, detected bp } return &bpCompressionStepData{ operation: types.Compress, - uploadedAlgorithm: uploadedAlgorithm, + uploadedAlgorithm: chosenFormat, uploadedAnnotations: annotations, srcCompressorName: detected.srcCompressorName, - uploadedCompressorName: uploadedAlgorithm.Name(), + uploadedCompressorName: chosenFormat.Name(), closers: []io.Closer{reader}, }, nil } @@ -135,7 +136,8 @@ func (ic *imageCopier) bpcCompressUncompressed(stream *sourceStream, detected bp // bpcRecompressCompressed checks if we should be recompressing a compressed input to another format, and returns a *bpCompressionStepData if so. func (ic *imageCopier) bpcRecompressCompressed(stream *sourceStream, detected bpDetectCompressionStepData) (*bpCompressionStepData, error) { if ic.c.dest.DesiredLayerCompression() == types.Compress && detected.isCompressed && - ic.c.compressionFormat != nil && ic.c.compressionFormat.Name() != detected.format.Name() { + ic.c.compressionFormat != nil && ic.c.compressionFormat.Name() != detected.format.Name() && + ic.src.CanCompressLayer(stream.info.MediaType, ic.c.compressionFormat) { // When the blob is compressed, but the desired format is different, it first needs to be decompressed and finally // re-compressed using the desired format. logrus.Debugf("Blob will be converted") @@ -173,7 +175,8 @@ func (ic *imageCopier) bpcRecompressCompressed(stream *sourceStream, detected bp // bpcDecompressCompressed checks if we should be decompressing a compressed input, and returns a *bpCompressionStepData if so. func (ic *imageCopier) bpcDecompressCompressed(stream *sourceStream, detected bpDetectCompressionStepData) (*bpCompressionStepData, error) { - if ic.c.dest.DesiredLayerCompression() == types.Decompress && detected.isCompressed { + if ic.c.dest.DesiredLayerCompression() == types.Decompress && detected.isCompressed && + ic.src.CanDecompressLayer(stream.info.MediaType) { logrus.Debugf("Blob will be decompressed") s, err := detected.decompressor(stream.reader) if err != nil { diff --git a/internal/image/docker_schema1.go b/internal/image/docker_schema1.go index dcbff71860..1f52ccff46 100644 --- a/internal/image/docker_schema1.go +++ b/internal/image/docker_schema1.go @@ -5,6 +5,7 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/manifest" + compression "github.com/containers/image/v5/pkg/compression/types" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -257,3 +258,18 @@ func (m *manifestSchema1) CanSubstituteLayers() bool { func (m *manifestSchema1) CanChangeLayerCompression(mimeType string) bool { return true // There are no MIME types in the manifest, so we must assume a valid image. } + +// CanDecompressLayer returns true if a layer with mimeType can be decompressed (and the code can handle that) +// FIXME FIXME: Some other API +// FIXME FIXME: Tests +func (m *manifestSchema1) CanDecompressLayer(mimeType string) bool { + return true // There are no MIME types in the manifest, so we must assume a valid image. +} + +// CanDecompressLayer returns true if a layer with mimeType can be (re)compressed to algorithm (and the code can handle that) +// FIXME FIXME: Some other API +// FIXME FIXME: Tests +func (m *manifestSchema1) CanCompressLayer(mimeType string, algorithm *compression.Algorithm) bool { + // There are no MIME types in the manifest, so we must assume a valid image. But gzip is the only supported format. + return algorithm.Name() == compression.GzipAlgorithmName +} diff --git a/internal/image/docker_schema2.go b/internal/image/docker_schema2.go index 5c3aec7f4c..4807e516e8 100644 --- a/internal/image/docker_schema2.go +++ b/internal/image/docker_schema2.go @@ -13,6 +13,7 @@ import ( "github.com/containers/image/v5/internal/iolimits" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/blobinfocache/none" + compression "github.com/containers/image/v5/pkg/compression/types" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -413,3 +414,17 @@ func (m *manifestSchema2) CanSubstituteLayers() bool { func (m *manifestSchema2) CanChangeLayerCompression(mimeType string) bool { return m.m.CanChangeLayerCompression(mimeType) } + +// CanDecompressLayer returns true if a layer with mimeType can be decompressed (and the code can handle that) +// FIXME FIXME: Some other API +// FIXME FIXME: Tests +func (m *manifestSchema2) CanDecompressLayer(mimeType string) bool { + return m.m.CanDecompressLayer(mimeType) +} + +// CanDecompressLayer returns true if a layer with mimeType can be (re)compressed to algorithm (and the code can handle that) +// FIXME FIXME: Some other API +// FIXME FIXME: Tests +func (m *manifestSchema2) CanCompressLayer(mimeType string, algorithm *compression.Algorithm) bool { + return m.m.CanCompressLayer(mimeType, algorithm) +} diff --git a/internal/image/manifest.go b/internal/image/manifest.go index 95b7109c0c..564efa0bef 100644 --- a/internal/image/manifest.go +++ b/internal/image/manifest.go @@ -6,6 +6,7 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/manifest" + compression "github.com/containers/image/v5/pkg/compression/types" "github.com/containers/image/v5/types" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -57,7 +58,14 @@ type genericManifest interface { // CanSubstituteBlobs returns true if the generic image copy code is allowed to substitute layers with their semantic equivalents. CanSubstituteLayers() bool // CanChangeLayerCompression returns true if it is allowed to compress/decompress layers with mimeType (and the code can handle that) + // FIXME FIXME: This does not say which compression formats are supported, and it almost certainly should. CanChangeLayerCompression(mimeType string) bool + // CanDecompressLayer returns true if a layer with mimeType can be decompressed (and the code can handle that) + // FIXME FIXME: Some other API + CanDecompressLayer(mimeType string) bool + // CanDecompressLayer returns true if a layer with mimeType can be (re)compressed to algorithm (and the code can handle that) + // FIXME FIXME: Some other API + CanCompressLayer(mimeType string, algorithm *compression.Algorithm) bool } // manifestInstanceFromBlob returns a genericManifest implementation for (manblob, mt) in src. diff --git a/internal/image/oci.go b/internal/image/oci.go index 8882d1bae3..69dbc26315 100644 --- a/internal/image/oci.go +++ b/internal/image/oci.go @@ -10,6 +10,7 @@ import ( internalManifest "github.com/containers/image/v5/internal/manifest" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/blobinfocache/none" + compression "github.com/containers/image/v5/pkg/compression/types" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -272,3 +273,17 @@ func (m *manifestOCI1) CanSubstituteLayers() bool { func (m *manifestOCI1) CanChangeLayerCompression(mimeType string) bool { return m.m.CanChangeLayerCompression(mimeType) } + +// CanDecompressLayer returns true if a layer with mimeType can be decompressed (and the code can handle that) +// FIXME FIXME: Some other API +// FIXME FIXME: Tests +func (m *manifestOCI1) CanDecompressLayer(mimeType string) bool { + return m.m.CanDecompressLayer(mimeType) +} + +// CanDecompressLayer returns true if a layer with mimeType can be (re)compressed to algorithm (and the code can handle that) +// FIXME FIXME: Some other API +// FIXME FIXME: Tests +func (m *manifestOCI1) CanCompressLayer(mimeType string, algorithm *compression.Algorithm) bool { + return m.m.CanCompressLayer(mimeType, algorithm) +} diff --git a/manifest/docker_schema2.go b/manifest/docker_schema2.go index 3625699273..d1c02c3351 100644 --- a/manifest/docker_schema2.go +++ b/manifest/docker_schema2.go @@ -301,3 +301,19 @@ func (m *Schema2) ImageID([]digest.Digest) (string, error) { func (m *Schema2) CanChangeLayerCompression(mimeType string) bool { return compressionVariantsRecognizeMIMEType(schema2CompressionMIMETypeSets, mimeType) } + +// CanDecompressLayer returns true if a layer with mimeType can be decompressed (and the code can handle that) +// FIXME FIXME: Some other API +// FIXME FIXME: Tests +func (m *Schema2) CanDecompressLayer(mimeType string) bool { + _, err := compressionVariantMIMEType(schema2CompressionMIMETypeSets, mimeType, nil) + return err == nil +} + +// CanDecompressLayer returns true if a layer with mimeType can be (re)compressed to algorithm (and the code can handle that) +// FIXME FIXME: Some other API +// FIXME FIXME: Tests +func (m *Schema2) CanCompressLayer(mimeType string, algorithm *compressiontypes.Algorithm) bool { + _, err := compressionVariantMIMEType(schema2CompressionMIMETypeSets, mimeType, algorithm) + return err == nil +} diff --git a/manifest/oci.go b/manifest/oci.go index ff7569fa98..b42fd7c5db 100644 --- a/manifest/oci.go +++ b/manifest/oci.go @@ -258,3 +258,25 @@ func (m *OCI1) CanChangeLayerCompression(mimeType string) bool { } return compressionVariantsRecognizeMIMEType(oci1CompressionMIMETypeSets, mimeType) } + +// CanDecompressLayer returns true if a layer with mimeType can be decompressed (and the code can handle that) +// FIXME FIXME: Some other API +// FIXME FIXME: Tests +func (m *OCI1) CanDecompressLayer(mimeType string) bool { + if m.Config.MediaType != imgspecv1.MediaTypeImageConfig { + return false + } + _, err := compressionVariantMIMEType(oci1CompressionMIMETypeSets, mimeType, nil) + return err == nil +} + +// CanDecompressLayer returns true if a layer with mimeType can be (re)compressed to algorithm (and the code can handle that) +// FIXME FIXME: Some other API +// FIXME FIXME: Tests +func (m *OCI1) CanCompressLayer(mimeType string, algorithm *compressiontypes.Algorithm) bool { + if m.Config.MediaType != imgspecv1.MediaTypeImageConfig { + return false + } + _, err := compressionVariantMIMEType(oci1CompressionMIMETypeSets, mimeType, algorithm) + return err == nil +}