From 9ba5b87db7c223f77d6ce22a1a60c144486194ee Mon Sep 17 00:00:00 2001 From: Romain Quinio Date: Sat, 30 Jul 2022 16:52:24 +0200 Subject: [PATCH] Support pulling base image layer compressed with zstd - Use commons-compress automatic algorithm detection via magic-bytes - Use optional dependency zstd-jni from commons-compress to support application/vnd.oci.image.layer.v1.tar+zstd layers --- build.gradle | 1 + docs/faq.md | 15 +++++- jib-core/build.gradle | 10 ++++ .../tools/jib/cache/CacheStorageWriter.java | 13 +++-- .../jib/cache/CacheStorageWriterTest.java | 48 ++++++++++++++++--- 5 files changed, 75 insertions(+), 12 deletions(-) diff --git a/build.gradle b/build.gradle index b73c6a580cf..de3013ba6ee 100644 --- a/build.gradle +++ b/build.gradle @@ -29,6 +29,7 @@ project.ext.dependencyStrings = [ MAVEN_EXTENSION: 'com.google.cloud.tools:jib-maven-plugin-extension-api:0.4.0', COMMONS_COMPRESS: 'org.apache.commons:commons-compress:1.21', + ZSTD_JNI: 'com.github.luben:zstd-jni:1.5.2-3', COMMONS_TEXT: 'org.apache.commons:commons-text:1.9', JACKSON_DATABIND: 'com.fasterxml.jackson.core:jackson-databind:2.13.3', JACKSON_DATAFORMAT_YAML: 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.3', diff --git a/docs/faq.md b/docs/faq.md index 5fe16d4cb67..2f18053c85c 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -44,7 +44,8 @@ If a question you have is not answered below, please [submit an issue](/../../is [How can I examine network traffic?](#how-can-i-examine-network-traffic)\ [How do I view debug logs for Jib?](#how-do-i-view-debug-logs-for-jib)\ [I am seeing `Method Not Found` or `Class Not Found` errors when building.](#i-am-seeing-method-not-found-or-class-not-found-errors-when-building)\ -[I am seeing `Unsupported class file major version` when building.](#i-am-seeing-unsupported-class-file-major-version-when-building) +[I am seeing `Unsupported class file major version` when building.](#i-am-seeing-unsupported-class-file-major-version-when-building)\ +[I am seeing `NoClassDefFoundError: com/github/luben/zstd/ZstdOutputStream` when building.](#i-am-seeing-noclassdeffounderror-com-github-luben-zstd-zstdoutputstream-when-building) **Launch Problems**\ [I am seeing `ImagePullBackoff` on my pods.](#i-am-seeing-imagepullbackoff-on-my-pods-in-minikube)\ @@ -764,6 +765,18 @@ Jib uses the [ASM library](https://asm.ow2.io/) to examine compiled Java bytecod Note that although the ASM library is the common cause of this error coming from Jib, it may be due to other reasons. Always check the full stack (`-e` or `-X` for Maven and `--stacktrace` for Gradle) to see where the error is coming from. +### I am seeing `NoClassDefFoundError: com/github/luben/zstd/ZstdOutputStream` when building. + +Jib supports base image layers with media-type `application/vnd.oci.image.layer.v1.tar+zstd`, i.e. compressed with zstd algorithm instead of gzip. + +However, the dependency to zstd is optional, so pulling such layers will result in: + +``` +java.lang.NoClassDefFoundError: com/github/luben/zstd/ZstdOutputStream +at org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream. +``` + +This can be solved by adding a dependency to artifact `com.github.luben:zstd-jni:1.5.2-3` to the plugin. ## Launch problems diff --git a/jib-core/build.gradle b/jib-core/build.gradle index 2ccc61525e0..7d3ee40c53d 100644 --- a/jib-core/build.gradle +++ b/jib-core/build.gradle @@ -4,6 +4,14 @@ plugins { id 'eclipse' } +java { + // Feature to handle base image layers compressed with zstd instead of gzip + // Will need to re-assess the optional dependency when zstd becomes widely used + registerFeature('zstdSupport') { + usingSourceSet(sourceSets.main) + } +} + dependencies { api dependencyStrings.BUILD_PLAN implementation dependencyStrings.GOOGLE_HTTP_CLIENT @@ -11,6 +19,7 @@ dependencies { implementation dependencyStrings.GOOGLE_AUTH_LIBRARY_OAUTH2_HTTP implementation dependencyStrings.COMMONS_COMPRESS + zstdSupportImplementation dependencyStrings.ZSTD_JNI implementation dependencyStrings.GUAVA implementation dependencyStrings.JACKSON_DATABIND implementation dependencyStrings.JACKSON_DATATYPE_JSR310 @@ -22,6 +31,7 @@ dependencies { testImplementation dependencyStrings.MOCKITO_CORE testImplementation dependencyStrings.SLF4J_API testImplementation dependencyStrings.SYSTEM_RULES + testImplementation dependencyStrings.ZSTD_JNI integrationTestImplementation dependencyStrings.JBCRYPT } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java index 56e385cc0d2..89f364fdde4 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java @@ -50,9 +50,10 @@ import java.util.List; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import javax.annotation.Nullable; +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorStreamFactory; /** Writes to the default cache storage engine. */ class CacheStorageWriter { @@ -159,9 +160,13 @@ static void moveIfDoesNotExist(Path source, Path destination) throws IOException */ private static DescriptorDigest getDiffIdByDecompressingFile(Path compressedFile) throws IOException { - try (InputStream fileInputStream = - new BufferedInputStream(new GZIPInputStream(Files.newInputStream(compressedFile)))) { - return Digests.computeDigest(fileInputStream).getDigest(); + try (InputStream in = + CompressorStreamFactory.getSingleton() + .createCompressorInputStream( + new BufferedInputStream(Files.newInputStream(compressedFile)))) { + return Digests.computeDigest(in).getDigest(); + } catch (CompressorException e) { + throw new IOException(e); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java index de9280cbf9c..8262f906293 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java @@ -41,6 +41,7 @@ import com.google.common.io.Resources; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.OutputStream; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -50,8 +51,9 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorStreamFactory; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.junit.Assert; @@ -68,6 +70,7 @@ private static BlobDescriptor getDigest(Blob blob) throws IOException { } private static Blob compress(Blob blob) { + // Don't use GzipCompressorOutputStream, which has different defaults than GZIPOutputStream return Blobs.from( outputStream -> { try (GZIPOutputStream compressorStream = new GZIPOutputStream(outputStream)) { @@ -77,8 +80,28 @@ private static Blob compress(Blob blob) { false); } + private static Blob compress(Blob blob, String compressorName) { + return Blobs.from( + outputStream -> { + try (OutputStream compressorStream = + CompressorStreamFactory.getSingleton() + .createCompressorOutputStream(compressorName, outputStream)) { + blob.writeTo(compressorStream); + } catch (CompressorException e) { + throw new RuntimeException(e); + } + }, + false); + } + private static Blob decompress(Blob blob) throws IOException { - return Blobs.from(new GZIPInputStream(new ByteArrayInputStream(Blobs.writeToByteArray(blob)))); + try { + return Blobs.from( + CompressorStreamFactory.getSingleton() + .createCompressorInputStream(new ByteArrayInputStream(Blobs.writeToByteArray(blob)))); + } catch (CompressorException e) { + throw new IOException(e); + } } private static T loadJsonResource(String path, Class jsonClass) @@ -103,10 +126,20 @@ public void setUp() throws IOException { @Test public void testWriteCompressed() throws IOException { Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); + Blob compressedLayerBlob = compress(uncompressedLayerBlob); + CachedLayer cachedLayer = cacheStorageWriter.writeCompressed(compressedLayerBlob); - CachedLayer cachedLayer = cacheStorageWriter.writeCompressed(compress(uncompressedLayerBlob)); + verifyCachedLayer(cachedLayer, uncompressedLayerBlob, compressedLayerBlob); + } - verifyCachedLayer(cachedLayer, uncompressedLayerBlob); + @Test + public void testWriteZstdCompressed() throws IOException { + Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); + Blob compressedLayerBlob = compress(uncompressedLayerBlob, CompressorStreamFactory.ZSTANDARD); + + CachedLayer cachedLayer = cacheStorageWriter.writeCompressed(compressedLayerBlob); + + verifyCachedLayer(cachedLayer, uncompressedLayerBlob, compressedLayerBlob); } @Test @@ -117,7 +150,7 @@ public void testWriteUncompressed() throws IOException { CachedLayer cachedLayer = cacheStorageWriter.writeUncompressed(uncompressedLayerBlob, selector); - verifyCachedLayer(cachedLayer, uncompressedLayerBlob); + verifyCachedLayer(cachedLayer, uncompressedLayerBlob, compress(uncompressedLayerBlob)); // Verifies that the files are present. Path selectorFile = cacheStorageFiles.getSelectorFile(selector); @@ -354,9 +387,10 @@ public void testMoveIfDoesNotExist_exceptionAfterFailure() { assertThat(exception.getCause()).hasMessageThat().isEqualTo("foo"); } - private void verifyCachedLayer(CachedLayer cachedLayer, Blob uncompressedLayerBlob) + private void verifyCachedLayer( + CachedLayer cachedLayer, Blob uncompressedLayerBlob, Blob compressedLayerBlob) throws IOException { - BlobDescriptor layerBlobDescriptor = getDigest(compress(uncompressedLayerBlob)); + BlobDescriptor layerBlobDescriptor = getDigest(compressedLayerBlob); DescriptorDigest layerDiffId = getDigest(uncompressedLayerBlob).getDigest(); // Verifies cachedLayer is correct.