From 0b3ff8aad13e5c877b96480f88e2ea4b2f4c1f7b Mon Sep 17 00:00:00 2001 From: Romain Quinio Date: Thu, 28 Jul 2022 22:50:16 +0200 Subject: [PATCH] Support pulling OCI Image Index manifests - Add Accept header application/vnd.oci.image.index.v1+json during base image pull - Re-use the logic from V22ManifestListTemplate to select the target platform diget via a new interface ManifestListTemplate --- .../jib/builder/steps/PullBaseImageStep.java | 9 +- .../image/json/BuildableManifestTemplate.java | 2 +- .../jib/image/json/ManifestListTemplate.java | 38 ++++++++ .../jib/image/json/OciIndexTemplate.java | 89 +++++++++++++++++-- .../image/json/V22ManifestListTemplate.java | 11 +-- .../jib/registry/AbstractManifestPuller.java | 10 ++- .../builder/steps/PullBaseImageStepTest.java | 43 ++++++++- .../jib/cache/CacheStorageReaderTest.java | 2 +- .../jib/cache/CacheStorageWriterTest.java | 2 +- .../jib/image/json/OciIndexTemplateTest.java | 15 ++++ .../jib/registry/ManifestPullerTest.java | 3 +- .../core/json/ociindex_platforms.json | 28 ++++++ 12 files changed, 226 insertions(+), 26 deletions(-) create mode 100644 jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestListTemplate.java create mode 100644 jib-core/src/test/resources/core/json/ociindex_platforms.json diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index 7c3481faf7f..0f77aeae71d 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -41,6 +41,7 @@ import com.google.cloud.tools.jib.image.json.ImageMetadataTemplate; import com.google.cloud.tools.jib.image.json.JsonToImageTranslator; import com.google.cloud.tools.jib.image.json.ManifestAndConfigTemplate; +import com.google.cloud.tools.jib.image.json.ManifestListTemplate; import com.google.cloud.tools.jib.image.json.ManifestTemplate; import com.google.cloud.tools.jib.image.json.PlatformNotFoundInBaseImageException; import com.google.cloud.tools.jib.image.json.UnknownManifestFormatException; @@ -315,8 +316,7 @@ private List pullBaseImages( JsonToImageTranslator.toImage(imageManifest, containerConfig)); } - // TODO: support OciIndexTemplate once AbstractManifestPuller starts to accept it. - Verify.verify(manifestTemplate instanceof V22ManifestListTemplate); + Verify.verify(manifestTemplate instanceof ManifestListTemplate); List manifestsAndConfigs = new ArrayList<>(); ImmutableList.Builder images = ImmutableList.builder(); @@ -332,7 +332,7 @@ private List pullBaseImages( String manifestDigest = lookUpPlatformSpecificImageManifest( - (V22ManifestListTemplate) manifestTemplate, platform); + (ManifestListTemplate) manifestTemplate, platform); // TODO: pull multiple manifests (+ container configs) in parallel. ManifestAndDigest imageManifestAndDigest = registryClient.pullManifest(manifestDigest); progressDispatcher2.dispatchProgress(1); @@ -360,10 +360,9 @@ private List pullBaseImages( * Looks through a manifest list for the manifest matching the {@code platform} and returns the * digest of the first manifest it finds. */ - // TODO: support OciIndexTemplate once AbstractManifestPuller starts to accept it. @VisibleForTesting String lookUpPlatformSpecificImageManifest( - V22ManifestListTemplate manifestListTemplate, Platform platform) + ManifestListTemplate manifestListTemplate, Platform platform) throws UnlistedPlatformInManifestListException { EventHandlers eventHandlers = buildContext.getEventHandlers(); diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/BuildableManifestTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/BuildableManifestTemplate.java index 967d59e50ac..c61ff881dfb 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/BuildableManifestTemplate.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/BuildableManifestTemplate.java @@ -59,7 +59,7 @@ class ContentDescriptorTemplate implements JsonTemplate { /** Necessary for Jackson to create from JSON. */ @SuppressWarnings("unused") - private ContentDescriptorTemplate() {} + protected ContentDescriptorTemplate() {} public long getSize() { return size; diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestListTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestListTemplate.java new file mode 100644 index 00000000000..8ed4cfa520f --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestListTemplate.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.image.json; + +import java.util.List; + +/** + * Parent class for manifest lists. + * + * @see V22ManifestListTemplate Docker V2.2 format + * @see OciIndexTemplate OCI format + */ +public interface ManifestListTemplate extends ManifestTemplate { + + /** + * Returns a list of digests for a specific platform found in the manifest list. see + * https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list + * + * @param architecture the architecture of the target platform + * @param os the os of the target platform + * @return a list of matching digests + */ + List getDigestsForPlatform(String architecture, String os); +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/OciIndexTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/OciIndexTemplate.java index 6166b226096..4ab00a4eab7 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/OciIndexTemplate.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/OciIndexTemplate.java @@ -16,11 +16,17 @@ package com.google.cloud.tools.jib.image.json; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.json.JsonTemplate; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import java.util.ArrayList; import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import javax.annotation.Nullable; /** * JSON template for OCI archive "index.json" file. @@ -36,6 +42,10 @@ * "mediaType": "application/vnd.oci.image.manifest.v1+json", * "digest": "sha256:e684b1dceef404268f17d4adf7f755fd9912b8ae64864b3954a83ebb8aa628b3", * "size": 1132, + * "platform": { + * "architecture": "ppc64le", + * "os": "linux" + * }, * "annotations": { * "org.opencontainers.image.ref.name": "gcr.io/project/image:tag" * } @@ -47,7 +57,7 @@ * @see OCI Image * Index Specification */ -public class OciIndexTemplate implements ManifestTemplate { +public class OciIndexTemplate implements ManifestListTemplate { /** The OCI Index media type. */ public static final String MEDIA_TYPE = "application/vnd.oci.image.index.v1+json"; @@ -55,8 +65,7 @@ public class OciIndexTemplate implements ManifestTemplate { private final int schemaVersion = 2; private final String mediaType = MEDIA_TYPE; - private final List manifests = - new ArrayList<>(); + private final List manifests = new ArrayList<>(); @Override public int getSchemaVersion() { @@ -75,8 +84,8 @@ public String getManifestMediaType() { * @param imageReferenceName the image reference name */ public void addManifest(BlobDescriptor descriptor, String imageReferenceName) { - BuildableManifestTemplate.ContentDescriptorTemplate contentDescriptorTemplate = - new BuildableManifestTemplate.ContentDescriptorTemplate( + ManifestDescriptorTemplate contentDescriptorTemplate = + new ManifestDescriptorTemplate( OciManifestTemplate.MANIFEST_MEDIA_TYPE, descriptor.getSize(), descriptor.getDigest()); contentDescriptorTemplate.setAnnotations( ImmutableMap.of("org.opencontainers.image.ref.name", imageReferenceName)); @@ -84,7 +93,75 @@ public void addManifest(BlobDescriptor descriptor, String imageReferenceName) { } @VisibleForTesting - public List getManifests() { + public List getManifests() { return manifests; } + + @Override + public List getDigestsForPlatform(String architecture, String os) { + return getManifests().stream() + .filter( + manifest -> + manifest.platform != null + && os.equals(manifest.platform.os) + && architecture.equals(manifest.platform.architecture)) + .map(ManifestDescriptorTemplate::getDigest) + .filter(Objects::nonNull) + .map(DescriptorDigest::toString) + .collect(Collectors.toList()); + } + + /** + * Template for inner JSON object representing a single platform specific manifest. See OCI Image Index + * Specification + */ + public static class ManifestDescriptorTemplate + extends BuildableManifestTemplate.ContentDescriptorTemplate { + + ManifestDescriptorTemplate(String mediaType, long size, DescriptorDigest digest) { + super(mediaType, size, digest); + } + + /** Necessary for Jackson to create from JSON. */ + @SuppressWarnings("unused") + private ManifestDescriptorTemplate() { + super(); + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Platform implements JsonTemplate { + @Nullable private String architecture; + @Nullable private String os; + + @Nullable + public String getArchitecture() { + return architecture; + } + + @Nullable + public String getOs() { + return os; + } + } + + @Nullable private OciIndexTemplate.ManifestDescriptorTemplate.Platform platform; + + /** + * Sets a platform. + * + * @param architecture the manifest architecture + * @param os the manifest os + */ + public void setPlatform(String architecture, String os) { + platform = new OciIndexTemplate.ManifestDescriptorTemplate.Platform(); + platform.architecture = architecture; + platform.os = os; + } + + @Nullable + public OciIndexTemplate.ManifestDescriptorTemplate.Platform getPlatform() { + return platform; + } + } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplate.java index c56e6c8859c..3293587f915 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplate.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplate.java @@ -63,7 +63,7 @@ * @see Image Manifest * Version 2, Schema 2: Manifest List */ -public class V22ManifestListTemplate implements ManifestTemplate { +public class V22ManifestListTemplate implements ManifestListTemplate { public static final String MANIFEST_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.list.v2+json"; @@ -101,14 +101,7 @@ public List getManifests() { return Preconditions.checkNotNull(manifests); } - /** - * Returns a list of digests for a specific platform found in the manifest list. see - * https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list - * - * @param architecture the architecture of the target platform - * @param os the os of the target platform - * @return a list of matching digests - */ + @Override public List getDigestsForPlatform(String architecture, String os) { return getManifests().stream() .filter( diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/AbstractManifestPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/AbstractManifestPuller.java index 1bb7522580f..1845b63bbf3 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/AbstractManifestPuller.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/AbstractManifestPuller.java @@ -80,12 +80,16 @@ public List getAccept() { if (manifestTemplateClass.equals(V22ManifestListTemplate.class)) { return Collections.singletonList(V22ManifestListTemplate.MANIFEST_MEDIA_TYPE); } + if (manifestTemplateClass.equals(OciIndexTemplate.class)) { + return Collections.singletonList(OciIndexTemplate.MEDIA_TYPE); + } return Arrays.asList( OciManifestTemplate.MANIFEST_MEDIA_TYPE, V22ManifestTemplate.MANIFEST_MEDIA_TYPE, V21ManifestTemplate.MEDIA_TYPE, - V22ManifestListTemplate.MANIFEST_MEDIA_TYPE); + V22ManifestListTemplate.MANIFEST_MEDIA_TYPE, + OciIndexTemplate.MEDIA_TYPE); } /** Parses the response body into a {@link ManifestAndDigest}. */ @@ -174,6 +178,10 @@ private T getManifestTemplateFromJson(String jsonString) return manifestTemplateClass.cast( JsonTemplateMapper.readJson(jsonString, V22ManifestListTemplate.class)); } + if (OciIndexTemplate.MEDIA_TYPE.equals(mediaType)) { + return manifestTemplateClass.cast( + JsonTemplateMapper.readJson(jsonString, OciIndexTemplate.class)); + } throw new UnknownManifestFormatException("Unknown mediaType: " + mediaType); } throw new UnknownManifestFormatException( diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStepTest.java index d23caf0b541..b0b8f0fa77f 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStepTest.java @@ -38,6 +38,7 @@ import com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate; import com.google.cloud.tools.jib.image.json.ImageMetadataTemplate; import com.google.cloud.tools.jib.image.json.ManifestAndConfigTemplate; +import com.google.cloud.tools.jib.image.json.OciIndexTemplate; import com.google.cloud.tools.jib.image.json.PlatformNotFoundInBaseImageException; import com.google.cloud.tools.jib.image.json.UnlistedPlatformInManifestListException; import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; @@ -202,7 +203,7 @@ public void testCall_offlineMode_cached() } @Test - public void testLookUpPlatformSpecificImageManifest() + public void testLookUpPlatformSpecificDockerImageManifest() throws IOException, UnlistedPlatformInManifestListException { String manifestListJson = " {\n" @@ -241,6 +242,46 @@ public void testLookUpPlatformSpecificImageManifest() "sha256:2222222222222222222222222222222222222222222222222222222222222222", manifestDigest); } + @Test + public void testLookUpPlatformSpecificOciManifest() + throws IOException, UnlistedPlatformInManifestListException { + String manifestListJson = + " {\n" + + " \"schemaVersion\": 2,\n" + + " \"mediaType\": \"application/vnd.oci.image.index.v1+json\",\n" + + " \"manifests\": [\n" + + " {\n" + + " \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",\n" + + " \"size\": 424,\n" + + " \"digest\": \"sha256:1111111111111111111111111111111111111111111111111111111111111111\",\n" + + " \"platform\": {\n" + + " \"architecture\": \"arm64\",\n" + + " \"os\": \"linux\"\n" + + " }\n" + + " },\n" + + " {\n" + + " \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",\n" + + " \"size\": 425,\n" + + " \"digest\": \"sha256:2222222222222222222222222222222222222222222222222222222222222222\",\n" + + " \"platform\": {\n" + + " \"architecture\": \"targetArchitecture\",\n" + + " \"os\": \"targetOS\"\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"; + + OciIndexTemplate manifestList = + JsonTemplateMapper.readJson(manifestListJson, OciIndexTemplate.class); + + String manifestDigest = + pullBaseImageStep.lookUpPlatformSpecificImageManifest( + manifestList, new Platform("targetArchitecture", "targetOS")); + + Assert.assertEquals( + "sha256:2222222222222222222222222222222222222222222222222222222222222222", manifestDigest); + } + @Test public void testGetCachedBaseImages_emptyCache() throws InvalidImageReferenceException, IOException, CacheCorruptedException, diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java index ca6a663e4a8..02575bb2079 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java @@ -297,7 +297,7 @@ public void testRetrieveMetadata_ociImageIndex() MatcherAssert.assertThat( metadata.getManifestList(), CoreMatchers.instanceOf(OciIndexTemplate.class)); - List manifestDescriptors = + List manifestDescriptors = ((OciIndexTemplate) metadata.getManifestList()).getManifests(); Assert.assertEquals(1, manifestDescriptors.size()); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java index de9280cbf9c..3473e37792d 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java @@ -291,7 +291,7 @@ public void testWriteMetadata_oci() MatcherAssert.assertThat( savedMetadata.getManifestList(), CoreMatchers.instanceOf(OciIndexTemplate.class)); - List savedManifestDescriptors = + List savedManifestDescriptors = ((OciIndexTemplate) savedMetadata.getManifestList()).getManifests(); Assert.assertEquals(1, savedManifestDescriptors.size()); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OciIndexTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OciIndexTemplateTest.java index 7f2724814e2..e8f189428de 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OciIndexTemplateTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OciIndexTemplateTest.java @@ -71,4 +71,19 @@ public void testFromJson() throws IOException, URISyntaxException, DigestExcepti "regis.try/repo:tag", manifest.getAnnotations().get("org.opencontainers.image.ref.name")); Assert.assertEquals(1000, manifest.getSize()); } + + @Test + public void testFromJsonWithPlatform() throws IOException, URISyntaxException, DigestException { + // Loads the JSON string. + Path jsonFile = Paths.get(Resources.getResource("core/json/ociindex_platforms.json").toURI()); + + // Deserializes into a manifest JSON object. + OciIndexTemplate ociIndexJson = + JsonTemplateMapper.readJsonFromFile(jsonFile, OciIndexTemplate.class); + + Assert.assertEquals(2, ociIndexJson.getManifests().size()); + Assert.assertEquals( + "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + ociIndexJson.getDigestsForPlatform("ppc64le", "linux").get(0)); + } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java index 342eb6935c7..2d9e6df74ea 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java @@ -318,7 +318,8 @@ public void testGetAccept() { OciManifestTemplate.MANIFEST_MEDIA_TYPE, V22ManifestTemplate.MANIFEST_MEDIA_TYPE, V21ManifestTemplate.MEDIA_TYPE, - V22ManifestListTemplate.MANIFEST_MEDIA_TYPE), + V22ManifestListTemplate.MANIFEST_MEDIA_TYPE, + OciIndexTemplate.MEDIA_TYPE), testManifestPuller.getAccept()); Assert.assertEquals( diff --git a/jib-core/src/test/resources/core/json/ociindex_platforms.json b/jib-core/src/test/resources/core/json/ociindex_platforms.json new file mode 100644 index 00000000000..f48cebce2de --- /dev/null +++ b/jib-core/src/test/resources/core/json/ociindex_platforms.json @@ -0,0 +1,28 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7143, + "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f", + "platform": { + "architecture": "ppc64le", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 7682, + "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270", + "platform": { + "architecture": "amd64", + "os": "linux" + } + } + ], + "annotations": { + "com.example.key1": "value1", + "com.example.key2": "value2" + } +} \ No newline at end of file