Handle gzip compression in Storage Emulator

CHANGELOG.md

@@ -3,3 +3,4 @@
 - Changes `superstatic` dependency to `v8`, addressing Hosting emulator issues on Windows.
 - Fixes internal library that was not being correctly published.
 - Adds `--disable-triggers` flag to RTDB write commands.
+- Fixes gzipped file handling in Storage Emulator.

scripts/storage-emulator-integration/conformance/firebase.endpoints.test.ts

@@ -3,6 +3,7 @@ import { expect } from "chai";
 import * as admin from "firebase-admin";
 import * as fs from "fs";
 import * as supertest from "supertest";
+import { gunzipSync } from "zlib";
 import { TEST_ENV } from "./env";
 import { EmulatorEndToEndTest } from "../../integration-helpers/framework";
 import {
@@ -343,7 +344,6 @@ describe("Firebase Storage endpoint conformance tests", () => {
           })
           .expect(200)
           .then((res) => {
-            console.log(res);
             return new URL(res.header["x-goog-upload-url"]);
           });
 
@@ -475,6 +475,77 @@ describe("Firebase Storage endpoint conformance tests", () => {
     });
   });
 
+  describe("gzip", () => {
+    it("should serve gunzipped file by default", async () => {
+      const contents = Buffer.from("hello world");
+      const fileName = "gzippedFile";
+      const file = testBucket.file(fileName);
+      await file.save(contents, {
+        gzip: true,
+        contentType: "text/plain",
+      });
+
+      // Use requestClient since supertest will decompress the response body by default.
+      await new Promise((resolve, reject) => {
+        TEST_ENV.requestClient.get(
+          `${firebaseHost}/v0/b/${storageBucket}/o/${fileName}?alt=media`,
+          { headers: { ...authHeader } },
+          (res) => {
+            expect(res.headers["content-encoding"]).to.be.undefined;
+            expect(res.headers["content-length"]).to.be.undefined;
+            expect(res.headers["content-type"]).to.be.eql("text/plain");
+
+            let responseBody = Buffer.alloc(0);
+            res
+              .on("data", (chunk) => {
+                responseBody = Buffer.concat([responseBody, chunk]);
+              })
+              .on("end", () => {
+                expect(responseBody).to.be.eql(contents);
+              })
+              .on("close", resolve)
+              .on("error", reject);
+          }
+        );
+      });
+    });
+
+    it("should serve gzipped file if Accept-Encoding header allows", async () => {
+      const contents = Buffer.from("hello world");
+      const fileName = "gzippedFile";
+      const file = testBucket.file(fileName);
+      await file.save(contents, {
+        gzip: true,
+        contentType: "text/plain",
+      });
+
+      // Use requestClient since supertest will decompress the response body by default.
+      await new Promise((resolve, reject) => {
+        TEST_ENV.requestClient.get(
+          `${firebaseHost}/v0/b/${storageBucket}/o/${fileName}?alt=media`,
+          { headers: { ...authHeader, "Accept-Encoding": "gzip" } },
+          (res) => {
+            expect(res.headers["content-encoding"]).to.be.eql("gzip");
+            expect(res.headers["content-type"]).to.be.eql("text/plain");
+
+            let responseBody = Buffer.alloc(0);
+            res
+              .on("data", (chunk) => {
+                responseBody = Buffer.concat([responseBody, chunk]);
+              })
+              .on("end", () => {
+                expect(responseBody).to.not.be.eql(contents);
+                const decompressed = gunzipSync(responseBody);
+                expect(decompressed).to.be.eql(contents);
+              })
+              .on("close", resolve)
+              .on("error", reject);
+          }
+        );
+      });
+    });
+  });
+
   describe("tokens", () => {
     beforeEach(async () => {
       await testBucket.upload(smallFilePath, { destination: TEST_FILE_NAME });

scripts/storage-emulator-integration/conformance/gcs-js-sdk.test.ts

@@ -13,6 +13,7 @@ import {
   TEST_SETUP_TIMEOUT,
   getTmpDir,
 } from "../utils";
+import { gunzipSync } from "zlib";
 
 // Test case that should only run when targeting the emulator.
 // Example use: emulatorOnly.it("Local only test case", () => {...});
@@ -109,14 +110,6 @@ describe("GCS Javascript SDK conformance tests", () => {
         fs.unlinkSync(content2);
       });
 
-      it("should handle gzip'd uploads", async () => {
-        // This appears to pass, but the file gets corrupted cause it's gzipped?
-        // expect(true).to.be.false;
-        await testBucket.upload(smallFilePath, {
-          gzip: true,
-        });
-      });
-
       it("should upload with provided metadata", async () => {
         const metadata = {
           contentDisposition: "attachment",
@@ -392,6 +385,18 @@ describe("GCS Javascript SDK conformance tests", () => {
   });
 
   describe(".file()", () => {
+    describe("#save()", () => {
+      it("should handle gzipped uploads", async () => {
+        const contents = "hello world";
+
+        const file = testBucket.file("gzippedFile");
+        await file.save(contents, { gzip: true, contentType: "text/plain" });
+
+        expect(file.metadata.contentType).to.be.eql("text/plain");
+        expect(file.metadata.contentEncoding).to.be.eql("gzip");
+      });
+    });
+
     describe("#exists()", () => {
       it("should return false for a file that does not exist", async () => {
         // Ensure that the file exists on the bucket before deleting it
@@ -488,6 +493,29 @@ describe("GCS Javascript SDK conformance tests", () => {
         expect(err).to.have.property("code", 404);
         expect(err).not.have.nested.property("errors[0]");
       });
+
+      it("should decompress gzipped file", async () => {
+        const contents = Buffer.from("hello world");
+
+        const file = testBucket.file("gzippedFile");
+        await file.save(contents, { gzip: true });
+
+        const [downloadedContents] = await file.download();
+        expect(downloadedContents).to.be.eql(contents);
+      });
+
+      it("should serve gzipped file if decompress option specified", async () => {
+        const contents = Buffer.from("hello world");
+
+        const file = testBucket.file("gzippedFile");
+        await file.save(contents, { gzip: true });
+
+        const [downloadedContents] = await file.download({ decompress: false });
+        expect(downloadedContents).to.not.be.eql(contents);
+
+        const ungzippedContents = gunzipSync(downloadedContents);
+        expect(ungzippedContents).to.be.eql(contents);
+      });
     });
 
     describe("#copy()", () => {

scripts/storage-emulator-integration/conformance/gcs.endpoints.test.ts

@@ -4,6 +4,7 @@ import * as admin from "firebase-admin";
 import * as fs from "fs";
 import * as supertest from "supertest";
 import { EmulatorEndToEndTest } from "../../integration-helpers/framework";
+import { gunzipSync } from "zlib";
 import { TEST_ENV } from "./env";
 import {
   EMULATORS_SHUTDOWN_DELAY_MS,
@@ -293,4 +294,75 @@ describe("GCS endpoint conformance tests", () => {
       });
     });
   });
+
+  describe("Gzip", () => {
+    it("should serve gunzipped file by default", async () => {
+      const contents = Buffer.from("hello world");
+      const fileName = "gzippedFile";
+      const file = testBucket.file(fileName);
+      await file.save(contents, {
+        gzip: true,
+        contentType: "text/plain",
+      });
+
+      // Use requestClient since supertest will decompress the response body by default.
+      await new Promise((resolve, reject) => {
+        TEST_ENV.requestClient.get(
+          `${storageHost}/download/storage/v1/b/${storageBucket}/o/${fileName}?alt=media`,
+          { headers: { ...authHeader } },
+          (res) => {
+            expect(res.headers["content-encoding"]).to.be.undefined;
+            expect(res.headers["content-length"]).to.be.undefined;
+            expect(res.headers["content-type"]).to.be.eql("text/plain");
+
+            let responseBody = Buffer.alloc(0);
+            res
+              .on("data", (chunk) => {
+                responseBody = Buffer.concat([responseBody, chunk]);
+              })
+              .on("end", () => {
+                expect(responseBody).to.be.eql(contents);
+              })
+              .on("close", resolve)
+              .on("error", reject);
+          }
+        );
+      });
+    });
+
+    it("should serve gzipped file if Accept-Encoding header allows", async () => {
+      const contents = Buffer.from("hello world");
+      const fileName = "gzippedFile";
+      const file = testBucket.file(fileName);
+      await file.save(contents, {
+        gzip: true,
+        contentType: "text/plain",
+      });
+
+      // Use requestClient since supertest will decompress the response body by default.
+      await new Promise((resolve, reject) => {
+        TEST_ENV.requestClient.get(
+          `${storageHost}/download/storage/v1/b/${storageBucket}/o/${fileName}?alt=media`,
+          { headers: { ...authHeader, "Accept-Encoding": "gzip" } },
+          (res) => {
+            expect(res.headers["content-encoding"]).to.be.eql("gzip");
+            expect(res.headers["content-type"]).to.be.eql("text/plain");
+
+            let responseBody = Buffer.alloc(0);
+            res
+              .on("data", (chunk) => {
+                responseBody = Buffer.concat([responseBody, chunk]);
+              })
+              .on("end", () => {
+                expect(responseBody).to.not.be.eql(contents);
+                const decompressed = gunzipSync(responseBody);
+                expect(decompressed).to.be.eql(contents);
+              })
+              .on("close", resolve)
+              .on("error", reject);
+          }
+        );
+      });
+    });
+  });
 });

src/emulator/storage/apis/firebase.ts

@@ -1,10 +1,10 @@
 import { EmulatorLogger } from "../../emulatorLogger";
 import { Emulators } from "../../types";
 import * as uuid from "uuid";
-import { gunzipSync } from "zlib";
 import { IncomingMetadata, OutgoingFirebaseMetadata, StoredFileMetadata } from "../metadata";
 import { Request, Response, Router } from "express";
 import { StorageEmulator } from "../index";
+import { sendFileBytes } from "./shared";
 import { EmulatorRegistry } from "../../registry";
 import { parseObjectUploadMultipartRequest } from "../multipart";
 import { NotFoundError, ForbiddenError } from "../errors";
@@ -27,7 +27,7 @@ export function createFirebaseEndpoints(emulator: StorageEmulator): Router {
 
   if (process.env.STORAGE_EMULATOR_DEBUG) {
     firebaseStorageAPI.use((req, res, next) => {
-      console.log("--------------INCOMING REQUEST--------------");
+      console.log("--------------INCOMING FIREBASE REQUEST--------------");
       console.log(`${req.method.toUpperCase()} ${req.path}`);
       console.log("-- query:");
       console.log(JSON.stringify(req.query, undefined, 2));
@@ -121,29 +121,7 @@ export function createFirebaseEndpoints(emulator: StorageEmulator): Router {
 
     // Object data request
     if (req.query.alt === "media") {
-      const isGZipped = metadata.contentEncoding === "gzip";
-      if (isGZipped) {
-        data = gunzipSync(data);
-      }
-      res.setHeader("Accept-Ranges", "bytes");
-      res.setHeader("Content-Type", metadata.contentType || "application/octet-stream");
-      res.setHeader("Content-Disposition", metadata.contentDisposition || "inline");
-      setObjectHeaders(res, metadata, { "Content-Encoding": isGZipped ? "identity" : undefined });
-
-      const byteRange = req.range(data.byteLength, { combine: true });
-
-      if (Array.isArray(byteRange) && byteRange.type === "bytes" && byteRange.length > 0) {
-        const range = byteRange[0];
-        res.setHeader(
-          "Content-Range",
-          `${byteRange.type} ${range.start}-${range.end}/${data.byteLength}`
-        );
-        // Byte range requests are inclusive for start and end
-        res.status(206).end(data.slice(range.start, range.end + 1));
-      } else {
-        res.end(data);
-      }
-      return;
+      return sendFileBytes(metadata, data, req, res);
     }
 
     // Object metadata request
@@ -531,27 +509,16 @@ export function createFirebaseEndpoints(emulator: StorageEmulator): Router {
   return firebaseStorageAPI;
 }
 
-function setObjectHeaders(
-  res: Response,
-  metadata: StoredFileMetadata,
-  headerOverride: {
-    "Content-Encoding": string | undefined;
-  } = { "Content-Encoding": undefined }
-): void {
+function setObjectHeaders(res: Response, metadata: StoredFileMetadata): void {
   if (metadata.contentDisposition) {
     res.setHeader("Content-Disposition", metadata.contentDisposition);
   }
-
-  if (headerOverride["Content-Encoding"]) {
-    res.setHeader("Content-Encoding", headerOverride["Content-Encoding"]);
-  } else if (metadata.contentEncoding) {
+  if (metadata.contentEncoding) {
     res.setHeader("Content-Encoding", metadata.contentEncoding);
   }
-
   if (metadata.cacheControl) {
     res.setHeader("Cache-Control", metadata.cacheControl);
   }
-
   if (metadata.contentLanguage) {
     res.setHeader("Content-Language", metadata.contentLanguage);
   }