Fix c/image fails to pull OCI image with non-http(s):// urls

Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>

docker/docker_image_src.go

@@ -236,33 +236,46 @@ func (s *dockerImageSource) ensureManifestIsLoaded(ctx context.Context) error {
 	return nil
 }
 
-func (s *dockerImageSource) getExternalBlob(ctx context.Context, urls []string) (io.ReadCloser, int64, error) {
+// getExternalBlob returns the reader of the first available blob URL.
+// This returns a bool value (named "fallback") which indicates whether the caller can fallback to the pull of
+// the non-external blob (i.e. pull from the registry).
+// This parameter becomes true only if no url is supported by this function.
+// Do not pass zero-length "urls". This is considered a fatal error and returns an error without allowing fallback.
+func (s *dockerImageSource) getExternalBlob(ctx context.Context, urls []string) (_ io.ReadCloser, _ int64, fallback bool, _ error) {
 	var (
 		resp *http.Response
 		err  error
 	)
 	if len(urls) == 0 {
-		return nil, 0, errors.New("internal error: getExternalBlob called with no URLs")
+		return nil, 0, false, errors.New("internal error: getExternalBlob called with no URLs") // fatal error
 	}
-	for _, url := range urls {
+	noSupportedURL := true
+	for _, u := range urls {
+		if u, err := url.Parse(u); err != nil || (u.Scheme != "http" && u.Scheme != "https") {
+			continue // unsupported url. skip this url.
+		}
+		noSupportedURL = false
 		// NOTE: we must not authenticate on additional URLs as those
 		//       can be abused to leak credentials or tokens.  Please
 		//       refer to CVE-2020-15157 for more information.
-		resp, err = s.c.makeRequestToResolvedURL(ctx, http.MethodGet, url, nil, nil, -1, noAuth, nil)
+		resp, err = s.c.makeRequestToResolvedURL(ctx, http.MethodGet, u, nil, nil, -1, noAuth, nil)
 		if err == nil {
 			if resp.StatusCode != http.StatusOK {
-				err = errors.Errorf("error fetching external blob from %q: %d (%s)", url, resp.StatusCode, http.StatusText(resp.StatusCode))
+				err = errors.Errorf("error fetching external blob from %q: %d (%s)", u, resp.StatusCode, http.StatusText(resp.StatusCode))
 				logrus.Debug(err)
 				resp.Body.Close()
 				continue
 			}
 			break
 		}
 	}
+	if noSupportedURL {
+		return nil, 0, true, errors.New("no supported url is specified") // fallback to non-external blob
+	}
 	if err != nil {
-		return nil, 0, err
+		return nil, 0, false, err
 	}
-	return resp.Body, getBlobSize(resp), nil
+	return resp.Body, getBlobSize(resp), false, nil
 }
 
 func getBlobSize(resp *http.Response) int64 {
@@ -408,7 +421,12 @@ func (s *dockerImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo,
 // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
 func (s *dockerImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
 	if len(info.URLs) != 0 {
-		return s.getExternalBlob(ctx, info.URLs)
+		r, s, fallback, err := s.getExternalBlob(ctx, info.URLs)
+		if err == nil {
+			return r, s, nil
+		} else if !fallback {
+			return nil, 0, err
+		}
 	}
 
 	path := fmt.Sprintf(blobsPath, reference.Path(s.physicalRef.ref), info.Digest.String())

oci/layout/oci_src.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
 	"strconv"
 
@@ -113,7 +114,12 @@ func (s *ociImageSource) HasThreadSafeGetBlob() bool {
 // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
 func (s *ociImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
 	if len(info.URLs) != 0 {
-		return s.getExternalBlob(ctx, info.URLs)
+		r, s, fallback, err := s.getExternalBlob(ctx, info.URLs)
+		if err == nil {
+			return r, s, nil
+		} else if !fallback {
+			return nil, 0, err
+		}
 	}
 
 	path, err := s.ref.blobPath(info.Digest, s.sharedBlobDir)
@@ -140,36 +146,48 @@ func (s *ociImageSource) GetSignatures(ctx context.Context, instanceDigest *dige
 	return [][]byte{}, nil
 }
 
-func (s *ociImageSource) getExternalBlob(ctx context.Context, urls []string) (io.ReadCloser, int64, error) {
+// getExternalBlob returns the reader of the first available blob URL.
+// This returns a bool value (named "fallback") which indicates whether the caller can fallback to the pull of
+// the non-external blob (i.e. pull from the registry).
+// This parameter becomes true only if no url is supported by this function.
+// Do not pass zero-length "urls". This is considered a fatal error and returns an error without allowing fallback.
+func (s *ociImageSource) getExternalBlob(ctx context.Context, urls []string) (_ io.ReadCloser, _ int64, fallback bool, _ error) {
 	if len(urls) == 0 {
-		return nil, 0, errors.New("internal error: getExternalBlob called with no URLs")
+		return nil, 0, false, errors.New("internal error: getExternalBlob called with no URLs") // fatal error
 	}
 
 	errWrap := errors.New("failed fetching external blob from all urls")
-	for _, url := range urls {
-
-		req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	noSupportedURL := true
+	for _, u := range urls {
+		if u, err := url.Parse(u); err != nil || (u.Scheme != "http" && u.Scheme != "https") {
+			continue // unsupported url. skip this url.
+		}
+		noSupportedURL = false
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
 		if err != nil {
-			errWrap = errors.Wrapf(errWrap, "fetching %s failed %s", url, err.Error())
+			errWrap = errors.Wrapf(errWrap, "fetching %s failed %s", u, err.Error())
 			continue
 		}
 
 		resp, err := s.client.Do(req)
 		if err != nil {
-			errWrap = errors.Wrapf(errWrap, "fetching %s failed %s", url, err.Error())
+			errWrap = errors.Wrapf(errWrap, "fetching %s failed %s", u, err.Error())
 			continue
 		}
 
 		if resp.StatusCode != http.StatusOK {
 			resp.Body.Close()
-			errWrap = errors.Wrapf(errWrap, "fetching %s failed, response code not 200", url)
+			errWrap = errors.Wrapf(errWrap, "fetching %s failed, response code not 200", u)
 			continue
 		}
 
-		return resp.Body, getBlobSize(resp), nil
+		return resp.Body, getBlobSize(resp), false, nil
+	}
+	if noSupportedURL {
+		return nil, 0, true, errors.New("no supported url is specified") // fallback to non-external blob
 	}
 
-	return nil, 0, errWrap
+	return nil, 0, false, errWrap
 }
 
 // LayerInfosForCopy returns either nil (meaning the values in the manifest are fine), or updated values for the layer