Skip to content

Commit

Permalink
Merge pull request #1462 from mtrmac/private
Browse files Browse the repository at this point in the history
Establish internal/private.ImageDestination
  • Loading branch information
rhatdan committed Feb 11, 2022
2 parents eafd4cf + 57ff3ff commit 8d40d56
Show file tree
Hide file tree
Showing 8 changed files with 141 additions and 137 deletions.
14 changes: 7 additions & 7 deletions copy/copy.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
"github.com/containers/image/v5/image"
internalblobinfocache "github.com/containers/image/v5/internal/blobinfocache"
"github.com/containers/image/v5/internal/pkg/platform"
internalTypes "github.com/containers/image/v5/internal/types"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/blobinfocache"
"github.com/containers/image/v5/pkg/compression"
Expand Down Expand Up @@ -1233,9 +1233,9 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
// Note: the storage destination optimizes the committing of
// layers which requires passing the index of the layer.
// Hence, we need to special case and cast.
dest, ok := ic.c.dest.(internalTypes.ImageDestinationWithOptions)
dest, ok := ic.c.dest.(private.ImageDestination)
if ok {
options := internalTypes.TryReusingBlobOptions{
options := private.TryReusingBlobOptions{
Cache: ic.c.blobInfoCache,
CanSubstitute: ic.canSubstituteBlobs,
SrcRef: srcRef,
Expand Down Expand Up @@ -1286,8 +1286,8 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
// of the source file are not known yet and must be fetched.
// Attempt a partial only when the source allows to retrieve a blob partially and
// the destination has support for it.
imgSource, okSource := ic.c.rawSource.(internalTypes.ImageSourceSeekable)
imgDest, okDest := ic.c.dest.(internalTypes.ImageDestinationPartial)
imgSource, okSource := ic.c.rawSource.(private.ImageSourceSeekable)
imgDest, okDest := ic.c.dest.(private.ImageDestinationPartial)
if okSource && okDest && !diffIDIsNeeded {
if reused, blobInfo := func() (bool, types.BlobInfo) { // A scope for defer
bar := ic.c.createProgressBar(pool, true, srcInfo, "blob", "done")
Expand Down Expand Up @@ -1662,9 +1662,9 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
// Note: the storage destination optimizes the committing of layers
// which requires passing the index of the layer. Hence, we need to
// special case and cast.
dest, ok := c.dest.(internalTypes.ImageDestinationWithOptions)
dest, ok := c.dest.(private.ImageDestination)
if ok {
options := internalTypes.PutBlobOptions{
options := private.PutBlobOptions{
Cache: c.blobInfoCache,
IsConfig: isConfig,
EmptyLayer: emptyLayer,
Expand Down
6 changes: 3 additions & 3 deletions copy/progress_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"io"
"time"

internalTypes "github.com/containers/image/v5/internal/types"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/types"
)

Expand Down Expand Up @@ -84,14 +84,14 @@ func (r *progressReader) Read(p []byte) (int, error) {
// are received.
type imageSourceSeekableProxy struct {
// source is the seekable input to read from.
source internalTypes.ImageSourceSeekable
source private.ImageSourceSeekable
// progress is the chan where the total number of bytes read so far are reported.
progress chan int64
}

// GetBlobAt reads from the ImageSourceSeekable and report how many bytes were received
// to the progress chan.
func (s imageSourceSeekableProxy) GetBlobAt(ctx context.Context, bInfo types.BlobInfo, chunks []internalTypes.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
func (s imageSourceSeekableProxy) GetBlobAt(ctx context.Context, bInfo types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
rc, errs, err := s.source.GetBlobAt(ctx, bInfo, chunks)
if err == nil {
total := int64(0)
Expand Down
10 changes: 5 additions & 5 deletions docker/docker_image_src.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (

"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/iolimits"
internalTypes "github.com/containers/image/v5/internal/types"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/sysregistriesv2"
"github.com/containers/image/v5/types"
Expand Down Expand Up @@ -288,7 +288,7 @@ func (s *dockerImageSource) HasThreadSafeGetBlob() bool {
}

// splitHTTP200ResponseToPartial splits a 200 response in multiple streams as specified by the chunks
func splitHTTP200ResponseToPartial(streams chan io.ReadCloser, errs chan error, body io.ReadCloser, chunks []internalTypes.ImageSourceChunk) {
func splitHTTP200ResponseToPartial(streams chan io.ReadCloser, errs chan error, body io.ReadCloser, chunks []private.ImageSourceChunk) {
defer close(streams)
defer close(errs)
currentOffset := uint64(0)
Expand Down Expand Up @@ -322,7 +322,7 @@ func splitHTTP200ResponseToPartial(streams chan io.ReadCloser, errs chan error,
}

// handle206Response reads a 206 response and send each part as a separate ReadCloser to the streams chan.
func handle206Response(streams chan io.ReadCloser, errs chan error, body io.ReadCloser, chunks []internalTypes.ImageSourceChunk, mediaType string, params map[string]string) {
func handle206Response(streams chan io.ReadCloser, errs chan error, body io.ReadCloser, chunks []private.ImageSourceChunk, mediaType string, params map[string]string) {
defer close(streams)
defer close(errs)
if !strings.HasPrefix(mediaType, "multipart/") {
Expand Down Expand Up @@ -359,7 +359,7 @@ func handle206Response(streams chan io.ReadCloser, errs chan error, body io.Read

// GetBlobAt returns a stream for the specified blob.
// The specified chunks must be not overlapping and sorted by their offset.
func (s *dockerImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []internalTypes.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
func (s *dockerImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []private.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
headers := make(map[string][]string)

var rangeVals []string
Expand Down Expand Up @@ -401,7 +401,7 @@ func (s *dockerImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo,
return streams, errs, nil
case http.StatusBadRequest:
res.Body.Close()
return nil, nil, internalTypes.BadPartialRequestError{Status: res.Status}
return nil, nil, private.BadPartialRequestError{Status: res.Status}
default:
err := httpResponseToError(res, "Error fetching partial blob")
if err == nil {
Expand Down
10 changes: 6 additions & 4 deletions docker/docker_image_src_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@ import (
"strings"
"testing"

internalTypes "github.com/containers/image/v5/internal/types"
"github.com/containers/image/v5/internal/private"
"github.com/containers/image/v5/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

var _ private.ImageSourceSeekable = (*dockerImageSource)(nil)

func TestDockerImageSourceReference(t *testing.T) {
manifestPathRegex := regexp.MustCompile("^/v2/.*/manifests/latest$")

Expand Down Expand Up @@ -130,7 +132,7 @@ func TestSplitHTTP200ResponseToPartial(t *testing.T) {
defer body.Close()
streams := make(chan io.ReadCloser)
errs := make(chan error)
chunks := []internalTypes.ImageSourceChunk{
chunks := []private.ImageSourceChunk{
{Offset: 1, Length: 2},
{Offset: 4, Length: 1},
}
Expand All @@ -150,7 +152,7 @@ func TestHandle206Response(t *testing.T) {
defer body.Close()
streams := make(chan io.ReadCloser)
errs := make(chan error)
chunks := []internalTypes.ImageSourceChunk{
chunks := []private.ImageSourceChunk{
{Offset: 1, Length: 2},
{Offset: 4, Length: 1},
}
Expand All @@ -171,7 +173,7 @@ func TestHandle206Response(t *testing.T) {
defer body.Close()
streams = make(chan io.ReadCloser)
errs = make(chan error)
chunks = []internalTypes.ImageSourceChunk{{Offset: 100, Length: 5}}
chunks = []private.ImageSourceChunk{{Offset: 100, Length: 5}}
mediaType = "text/plain"
params = map[string]string{}
go handle206Response(streams, errs, body, chunks, mediaType, params)
Expand Down
90 changes: 90 additions & 0 deletions internal/private/private.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package private

import (
"context"
"io"

"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/types"
)

// ImageDestination is an internal extension to the types.ImageDestination
// interface.
type ImageDestination interface {
types.ImageDestination

// PutBlobWithOptions writes contents of stream and returns data representing the result.
// inputInfo.Digest can be optionally provided if known; if provided, and stream is read to the end without error, the digest MUST match the stream contents.
// inputInfo.Size is the expected length of stream, if known.
// inputInfo.MediaType describes the blob format, if known.
// WARNING: The contents of stream are being verified on the fly. Until stream.Read() returns io.EOF, the contents of the data SHOULD NOT be available
// to any other readers for download using the supplied digest.
// If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far.
PutBlobWithOptions(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, options PutBlobOptions) (types.BlobInfo, error)

// TryReusingBlobWithOptions checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree).
// info.Digest must not be empty.
// If the blob has been successfully reused, returns (true, info, nil); info must contain at least a digest and size, and may
// include CompressionOperation and CompressionAlgorithm fields to indicate that a change to the compression type should be
// reflected in the manifest that will be written.
// If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure.
TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options TryReusingBlobOptions) (bool, types.BlobInfo, error)
}

// PutBlobOptions are used in PutBlobWithOptions.
type PutBlobOptions struct {
// Cache to look up blob infos.
Cache types.BlobInfoCache
// Denotes whether the blob is a config or not.
IsConfig bool
// Indicates an empty layer.
EmptyLayer bool
// The corresponding index in the layer slice.
LayerIndex *int
}

// TryReusingBlobOptions are used in TryReusingBlobWithOptions.
type TryReusingBlobOptions struct {
// Cache to look up blob infos.
Cache types.BlobInfoCache
// Use an equivalent of the desired blob.
CanSubstitute bool
// Indicates an empty layer.
EmptyLayer bool
// The corresponding index in the layer slice.
LayerIndex *int
// The reference of the image that contains the target blob.
SrcRef reference.Named
}

// ImageSourceChunk is a portion of a blob.
// This API is experimental and can be changed without bumping the major version number.
type ImageSourceChunk struct {
Offset uint64
Length uint64
}

// ImageSourceSeekable is an image source that permits to fetch chunks of the entire blob.
// This API is experimental and can be changed without bumping the major version number.
type ImageSourceSeekable interface {
// GetBlobAt returns a stream for the specified blob.
// The specified chunks must be not overlapping and sorted by their offset.
GetBlobAt(context.Context, types.BlobInfo, []ImageSourceChunk) (chan io.ReadCloser, chan error, error)
}

// ImageDestinationPartial is a service to store a blob by requesting the missing chunks to a ImageSourceSeekable.
// This API is experimental and can be changed without bumping the major version number.
type ImageDestinationPartial interface {
// PutBlobPartial writes contents of stream and returns data representing the result.
PutBlobPartial(ctx context.Context, stream ImageSourceSeekable, srcInfo types.BlobInfo, cache types.BlobInfoCache) (types.BlobInfo, error)
}

// BadPartialRequestError is returned by ImageSourceSeekable.GetBlobAt on an invalid request.
type BadPartialRequestError struct {
Status string
}

func (e BadPartialRequestError) Error() string {
return e.Status
}
91 changes: 0 additions & 91 deletions internal/types/types.go

This file was deleted.

0 comments on commit 8d40d56

Please sign in to comment.