Merge pull request #2696 from AL333Z/opt-in-fhcrc-in-gzip-compression

Opt-in for FHCRC in gzip compression

build.sbt

@@ -153,7 +153,20 @@ ThisBuild / mimaBinaryIssueFilters ++= Seq(
   ProblemFilters.exclude[MissingClassProblem]("fs2.Compiler$TargetLowPriority$MonadCancelTarget"),
   ProblemFilters.exclude[MissingClassProblem]("fs2.Compiler$TargetLowPriority$MonadErrorTarget"),
   ProblemFilters.exclude[MissingTypesProblem]("fs2.Compiler$TargetLowPriority$SyncTarget"),
-  ProblemFilters.exclude[MissingClassProblem]("fs2.Chunk$VectorChunk")
+  ProblemFilters.exclude[MissingClassProblem]("fs2.Chunk$VectorChunk"),
+  ProblemFilters.exclude[ReversedMissingMethodProblem](
+    "fs2.compression.DeflateParams.fhCrcEnabled"
+  ),
+  ProblemFilters.exclude[DirectMissingMethodProblem](
+    "fs2.compression.DeflateParams#DeflateParamsImpl.copy"
+  ),
+  ProblemFilters.exclude[DirectMissingMethodProblem](
+    "fs2.compression.DeflateParams#DeflateParamsImpl.this"
+  ),
+  ProblemFilters.exclude[MissingTypesProblem]("fs2.compression.DeflateParams$DeflateParamsImpl$"),
+  ProblemFilters.exclude[DirectMissingMethodProblem](
+    "fs2.compression.DeflateParams#DeflateParamsImpl.apply"
+  )
 )
 
 lazy val root = project

core/jvm/src/main/scala/fs2/compression/CompressionPlatform.scala

@@ -404,7 +404,8 @@ private[compression] trait CompressionCompanionPlatform {
                     fileName,
                     modificationTime,
                     comment,
-                    params.level.juzDeflaterLevel
+                    params.level.juzDeflaterLevel,
+                    params.fhCrcEnabled
                   ) ++
                     _deflate(
                       params,
@@ -425,7 +426,8 @@ private[compression] trait CompressionCompanionPlatform {
           fileName: Option[String],
           modificationTime: Option[Instant],
           comment: Option[String],
-          deflateLevel: Int
+          deflateLevel: Int,
+          fhCrcEnabled: Boolean
       ): Stream[F, Byte] = {
         // See RFC 1952: https://www.ietf.org/rfc/rfc1952.txt
         val secondsSince197001010000: Long =
@@ -434,7 +436,7 @@ private[compression] trait CompressionCompanionPlatform {
           gzipMagicFirstByte, // ID1: Identification 1
           gzipMagicSecondByte, // ID2: Identification 2
           gzipCompressionMethod.DEFLATE, // CM: Compression Method
-          (gzipFlag.FHCRC + // FLG: Header CRC
+          ((if (fhCrcEnabled) gzipFlag.FHCRC else zeroByte) + // FLG: Header CRC
             fileName.map(_ => gzipFlag.FNAME).getOrElse(zeroByte) + // FLG: File name
             comment.map(_ => gzipFlag.FCOMMENT).getOrElse(zeroByte)).toByte, // FLG: Comment
           (secondsSince197001010000 & 0xff).toByte, // MTIME: Modification Time
@@ -463,10 +465,16 @@ private[compression] trait CompressionCompanionPlatform {
           bytes
         }
         val crc32Value = crc32.getValue
-        val crc16 = Array[Byte](
-          (crc32Value & 0xff).toByte,
-          ((crc32Value >> 8) & 0xff).toByte
-        )
+
+        val crc16 =
+          if (fhCrcEnabled)
+            Array[Byte](
+              (crc32Value & 0xff).toByte,
+              ((crc32Value >> 8) & 0xff).toByte
+            )
+          else
+            Array.emptyByteArray
+
         Stream.chunk(moveAsChunkBytes(header)) ++
           fileNameEncoded
             .map(bytes => Stream.chunk(moveAsChunkBytes(bytes)) ++ Stream.emit(zeroByte))

core/jvm/src/test/scala/fs2/JvmCompressionSuite.scala

@@ -30,6 +30,8 @@ import java.nio.charset.StandardCharsets
 import java.time.Instant
 import java.util.zip._
 import scala.collection.mutable
+import scodec.bits.crc
+import scodec.bits.ByteVector
 
 class JvmCompressionSuite extends CompressionSuite {
 
@@ -246,6 +248,35 @@ class JvmCompressionSuite extends CompressionSuite {
       .map(compressed => assert(compressed.length < uncompressed.length))
   }
 
+  test("gzip.compresses input, with FLG.FHCRC set") {
+    Stream
+      .chunk[IO, Byte](Chunk.array(getBytes("Foo")))
+      .through(
+        Compression[IO].gzip(
+          fileName = None,
+          modificationTime = None,
+          comment = None,
+          deflateParams = DeflateParams.apply(
+            bufferSize = 1024 * 32,
+            header = ZLibParams.Header.GZIP,
+            level = DeflateParams.Level.DEFAULT,
+            strategy = DeflateParams.Strategy.DEFAULT,
+            flushMode = DeflateParams.FlushMode.DEFAULT,
+            fhCrcEnabled = true
+          )
+        )
+      )
+      .compile
+      .toVector
+      .map { compressed =>
+        val headerBytes = ByteVector(compressed.take(10))
+        val crc32 = crc.crc32(headerBytes.toBitVector).toByteArray
+        val expectedCrc16 = crc32.reverse.take(2).toVector
+        val actualCrc16 = compressed.drop(10).take(2)
+        assertEquals(actualCrc16, expectedCrc16)
+      }
+  }
+
   test("gunzip limit fileName and comment length") {
     val longString: String =
       Array

core/shared/src/main/scala/fs2/compression/DeflateParams.scala

@@ -45,6 +45,13 @@ sealed trait DeflateParams {
     */
   val flushMode: DeflateParams.FlushMode
 
+  /** A [[Boolean]] indicating whether the `FLG.FHCRC` bit is set. Default is `false`.
+    *  This is provided so that the compressor can be configured to have the CRC16 check enabled.
+    *  Why opt-in and not opt-out? It turned out not all clients implemented that right.
+    *  More context [[https://github.com/http4s/http4s/issues/5417 in this issue]].
+    */
+  val fhCrcEnabled: Boolean
+
   private[fs2] val bufferSizeOrMinimum: Int = bufferSize.max(128)
 }
 
@@ -57,14 +64,25 @@ object DeflateParams {
       strategy: DeflateParams.Strategy = DeflateParams.Strategy.DEFAULT,
       flushMode: DeflateParams.FlushMode = DeflateParams.FlushMode.DEFAULT
   ): DeflateParams =
-    DeflateParamsImpl(bufferSize, header, level, strategy, flushMode)
+    DeflateParamsImpl(bufferSize, header, level, strategy, flushMode, false)
+
+  def apply(
+      bufferSize: Int,
+      header: ZLibParams.Header,
+      level: DeflateParams.Level,
+      strategy: DeflateParams.Strategy,
+      flushMode: DeflateParams.FlushMode,
+      fhCrcEnabled: Boolean
+  ): DeflateParams =
+    DeflateParamsImpl(bufferSize, header, level, strategy, flushMode, fhCrcEnabled)
 
   private case class DeflateParamsImpl(
       bufferSize: Int,
       header: ZLibParams.Header,
       level: DeflateParams.Level,
       strategy: DeflateParams.Strategy,
-      flushMode: DeflateParams.FlushMode
+      flushMode: DeflateParams.FlushMode,
+      fhCrcEnabled: Boolean
   ) extends DeflateParams
 
   sealed abstract class Level(private[fs2] val juzDeflaterLevel: Int)