diff --git a/build.gradle b/build.gradle index b73c6a580c..de3013ba6e 100644 --- a/build.gradle +++ b/build.gradle @@ -29,6 +29,7 @@ project.ext.dependencyStrings = [ MAVEN_EXTENSION: 'com.google.cloud.tools:jib-maven-plugin-extension-api:0.4.0', COMMONS_COMPRESS: 'org.apache.commons:commons-compress:1.21', + ZSTD_JNI: 'com.github.luben:zstd-jni:1.5.2-3', COMMONS_TEXT: 'org.apache.commons:commons-text:1.9', JACKSON_DATABIND: 'com.fasterxml.jackson.core:jackson-databind:2.13.3', JACKSON_DATAFORMAT_YAML: 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.3', diff --git a/docs/faq.md b/docs/faq.md index 5fe16d4cb6..93da3c99b8 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -44,7 +44,8 @@ If a question you have is not answered below, please [submit an issue](/../../is [How can I examine network traffic?](#how-can-i-examine-network-traffic)\ [How do I view debug logs for Jib?](#how-do-i-view-debug-logs-for-jib)\ [I am seeing `Method Not Found` or `Class Not Found` errors when building.](#i-am-seeing-method-not-found-or-class-not-found-errors-when-building)\ -[I am seeing `Unsupported class file major version` when building.](#i-am-seeing-unsupported-class-file-major-version-when-building) +[I am seeing `Unsupported class file major version` when building.](#i-am-seeing-unsupported-class-file-major-version-when-building)\ +[I am seeing `NoClassDefFoundError: com/github/luben/zstd/ZstdOutputStream` when building.](#i-am-seeing-noclassdeffounderror-comgithublubenzstdzstdoutputstream-when-building) **Launch Problems**\ [I am seeing `ImagePullBackoff` on my pods.](#i-am-seeing-imagepullbackoff-on-my-pods-in-minikube)\ @@ -764,6 +765,18 @@ Jib uses the [ASM library](https://asm.ow2.io/) to examine compiled Java bytecod Note that although the ASM library is the common cause of this error coming from Jib, it may be due to other reasons. Always check the full stack (`-e` or `-X` for Maven and `--stacktrace` for Gradle) to see where the error is coming from. +### I am seeing `NoClassDefFoundError: com/github/luben/zstd/ZstdOutputStream` when building. + +Jib supports base image layers with media-type `application/vnd.oci.image.layer.v1.tar+zstd`, i.e. compressed with zstd algorithm instead of gzip. + +However, the dependency to zstd is optional, so pulling such layers will result in: + +``` +java.lang.NoClassDefFoundError: com/github/luben/zstd/ZstdOutputStream +at org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream. +``` + +This can be solved by adding a dependency to artifact `com.github.luben:zstd-jni:1.5.2-3` to the plugin. ## Launch problems diff --git a/jib-core/build.gradle b/jib-core/build.gradle index 2ccc61525e..7d3ee40c53 100644 --- a/jib-core/build.gradle +++ b/jib-core/build.gradle @@ -4,6 +4,14 @@ plugins { id 'eclipse' } +java { + // Feature to handle base image layers compressed with zstd instead of gzip + // Will need to re-assess the optional dependency when zstd becomes widely used + registerFeature('zstdSupport') { + usingSourceSet(sourceSets.main) + } +} + dependencies { api dependencyStrings.BUILD_PLAN implementation dependencyStrings.GOOGLE_HTTP_CLIENT @@ -11,6 +19,7 @@ dependencies { implementation dependencyStrings.GOOGLE_AUTH_LIBRARY_OAUTH2_HTTP implementation dependencyStrings.COMMONS_COMPRESS + zstdSupportImplementation dependencyStrings.ZSTD_JNI implementation dependencyStrings.GUAVA implementation dependencyStrings.JACKSON_DATABIND implementation dependencyStrings.JACKSON_DATATYPE_JSR310 @@ -22,6 +31,7 @@ dependencies { testImplementation dependencyStrings.MOCKITO_CORE testImplementation dependencyStrings.SLF4J_API testImplementation dependencyStrings.SYSTEM_RULES + testImplementation dependencyStrings.ZSTD_JNI integrationTestImplementation dependencyStrings.JBCRYPT } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java index 56e385cc0d..89f364fdde 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java @@ -50,9 +50,10 @@ import java.util.List; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import javax.annotation.Nullable; +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorStreamFactory; /** Writes to the default cache storage engine. */ class CacheStorageWriter { @@ -159,9 +160,13 @@ static void moveIfDoesNotExist(Path source, Path destination) throws IOException */ private static DescriptorDigest getDiffIdByDecompressingFile(Path compressedFile) throws IOException { - try (InputStream fileInputStream = - new BufferedInputStream(new GZIPInputStream(Files.newInputStream(compressedFile)))) { - return Digests.computeDigest(fileInputStream).getDigest(); + try (InputStream in = + CompressorStreamFactory.getSingleton() + .createCompressorInputStream( + new BufferedInputStream(Files.newInputStream(compressedFile)))) { + return Digests.computeDigest(in).getDigest(); + } catch (CompressorException e) { + throw new IOException(e); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java index de9280cbf9..5c851ef517 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java @@ -41,6 +41,7 @@ import com.google.common.io.Resources; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.OutputStream; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -50,8 +51,9 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; +import org.apache.commons.compress.compressors.CompressorException; +import org.apache.commons.compress.compressors.CompressorStreamFactory; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.junit.Assert; @@ -68,6 +70,7 @@ private static BlobDescriptor getDigest(Blob blob) throws IOException { } private static Blob compress(Blob blob) { + // Don't use GzipCompressorOutputStream, which has different defaults than GZIPOutputStream return Blobs.from( outputStream -> { try (GZIPOutputStream compressorStream = new GZIPOutputStream(outputStream)) { @@ -77,8 +80,28 @@ private static Blob compress(Blob blob) { false); } + private static Blob compress(Blob blob, String compressorName) { + return Blobs.from( + outputStream -> { + try (OutputStream compressorStream = + CompressorStreamFactory.getSingleton() + .createCompressorOutputStream(compressorName, outputStream)) { + blob.writeTo(compressorStream); + } catch (CompressorException e) { + throw new RuntimeException(e); + } + }, + false); + } + private static Blob decompress(Blob blob) throws IOException { - return Blobs.from(new GZIPInputStream(new ByteArrayInputStream(Blobs.writeToByteArray(blob)))); + try { + return Blobs.from( + CompressorStreamFactory.getSingleton() + .createCompressorInputStream(new ByteArrayInputStream(Blobs.writeToByteArray(blob)))); + } catch (CompressorException e) { + throw new IOException(e); + } } private static T loadJsonResource(String path, Class jsonClass) @@ -103,10 +126,27 @@ public void setUp() throws IOException { @Test public void testWriteCompressed() throws IOException { Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); + Blob compressedLayerBlob = compress(uncompressedLayerBlob); + CachedLayer cachedLayer = cacheStorageWriter.writeCompressed(compressedLayerBlob); - CachedLayer cachedLayer = cacheStorageWriter.writeCompressed(compress(uncompressedLayerBlob)); + verifyCachedLayer(cachedLayer, uncompressedLayerBlob, compressedLayerBlob); + } - verifyCachedLayer(cachedLayer, uncompressedLayerBlob); + @Test + public void testWriteZstdCompressed() throws IOException { + Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); + Blob compressedLayerBlob = compress(uncompressedLayerBlob, CompressorStreamFactory.ZSTANDARD); + + CachedLayer cachedLayer = cacheStorageWriter.writeCompressed(compressedLayerBlob); + + verifyCachedLayer(cachedLayer, uncompressedLayerBlob, compressedLayerBlob); + } + + @Test(expected = IOException.class) + public void testWriteCompressWhenUncompressed() throws IOException { + Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); + // The detection of compression algorithm will fail + cacheStorageWriter.writeCompressed(uncompressedLayerBlob); } @Test @@ -117,7 +157,7 @@ public void testWriteUncompressed() throws IOException { CachedLayer cachedLayer = cacheStorageWriter.writeUncompressed(uncompressedLayerBlob, selector); - verifyCachedLayer(cachedLayer, uncompressedLayerBlob); + verifyCachedLayer(cachedLayer, uncompressedLayerBlob, compress(uncompressedLayerBlob)); // Verifies that the files are present. Path selectorFile = cacheStorageFiles.getSelectorFile(selector); @@ -354,9 +394,10 @@ public void testMoveIfDoesNotExist_exceptionAfterFailure() { assertThat(exception.getCause()).hasMessageThat().isEqualTo("foo"); } - private void verifyCachedLayer(CachedLayer cachedLayer, Blob uncompressedLayerBlob) + private void verifyCachedLayer( + CachedLayer cachedLayer, Blob uncompressedLayerBlob, Blob compressedLayerBlob) throws IOException { - BlobDescriptor layerBlobDescriptor = getDigest(compress(uncompressedLayerBlob)); + BlobDescriptor layerBlobDescriptor = getDigest(compressedLayerBlob); DescriptorDigest layerDiffId = getDigest(uncompressedLayerBlob).getDigest(); // Verifies cachedLayer is correct.