Handle gzip compression in Storage Emulator (#5185)

* fix gzip for gcloud

* fix firebase

* revert

* lint

* update changelog

CHANGELOG.md

@@ -1,2 +1,3 @@
+- Fixes gzipped file handling in Storage Emulator.
 - Add support for object list using certain Admin SDKs (#5208)
 - Fixes source token expiration issue by acquiring new source token upon expiration.

scripts/storage-emulator-integration/conformance/env.ts

@@ -74,7 +74,7 @@ function readEmulatorConfig(): FrameworkOptions {
 class ConformanceTestEnvironment {
   private _prodAppConfig: any;
   private _emulatorConfig: any;
-  private _prodServiceAccountKeyJson?: any;
+  private _prodServiceAccountKeyJson?: any | null;
   private _adminAccessToken?: string;
 
   get useProductionServers() {
@@ -125,9 +125,10 @@ class ConformanceTestEnvironment {
   get prodServiceAccountKeyJson() {
     if (this._prodServiceAccountKeyJson === undefined) {
       const filePath = path.join(__dirname, TEST_CONFIG.prodServiceAccountKeyFilePath);
-      return TEST_CONFIG.prodServiceAccountKeyFilePath && fs.existsSync(filePath)
-        ? readAbsoluteJson(filePath)
-        : null;
+      this._prodServiceAccountKeyJson =
+        TEST_CONFIG.prodServiceAccountKeyFilePath && fs.existsSync(filePath)
+          ? readAbsoluteJson(filePath)
+          : null;
     }
     return this._prodServiceAccountKeyJson;
   }

scripts/storage-emulator-integration/conformance/firebase-js-sdk.test.ts

@@ -14,11 +14,14 @@ import {
   SMALL_FILE_SIZE,
   TEST_SETUP_TIMEOUT,
   getTmpDir,
-  writeToFile,
 } from "../utils";
 
 const TEST_FILE_NAME = "testing/storage_ref/testFile";
 
+// Test case that should only run when targeting the emulator.
+// Example use: emulatorOnly.it("Local only test case", () => {...});
+const emulatorOnly = { it: TEST_ENV.useProductionServers ? it.skip : it };
+
 describe("Firebase Storage JavaScript SDK conformance tests", () => {
   const storageBucket = TEST_ENV.appConfig.storageBucket;
   const expectedFirebaseHost = TEST_ENV.firebaseHost;
@@ -27,11 +30,6 @@ describe("Firebase Storage JavaScript SDK conformance tests", () => {
   const tmpDir = getTmpDir();
   const smallFilePath: string = createRandomFile("small_file", SMALL_FILE_SIZE, tmpDir);
   const emptyFilePath: string = createRandomFile("empty_file", 0, tmpDir);
-  const imageFilePath = writeToFile(
-    "image_base64",
-    Buffer.from(IMAGE_FILE_BASE64, "base64"),
-    tmpDir
-  );
 
   let test: EmulatorEndToEndTest;
   let testBucket: Bucket;
@@ -451,7 +449,8 @@ describe("Firebase Storage JavaScript SDK conformance tests", () => {
       });
 
       it("serves the right content", async () => {
-        await testBucket.upload(imageFilePath, { destination: TEST_FILE_NAME });
+        const contents = Buffer.from("hello world");
+        await testBucket.file(TEST_FILE_NAME).save(contents);
         await signInToFirebaseAuth(page);
 
         const downloadUrl = await page.evaluate((filename) => {
@@ -460,19 +459,21 @@ describe("Firebase Storage JavaScript SDK conformance tests", () => {
 
         await new Promise((resolve, reject) => {
           TEST_ENV.requestClient.get(downloadUrl, (response) => {
-            const data: any = [];
+            let data = Buffer.alloc(0);
             response
-              .on("data", (chunk) => data.push(chunk))
+              .on("data", (chunk) => {
+                data = Buffer.concat([data, chunk]);
+              })
               .on("end", () => {
-                expect(Buffer.concat(data)).to.deep.equal(Buffer.from(IMAGE_FILE_BASE64, "base64"));
+                expect(data).to.deep.equal(contents);
               })
               .on("close", resolve)
               .on("error", reject);
           });
         });
       });
 
-      it("serves content successfully when spammed with calls", async () => {
+      emulatorOnly.it("serves content successfully when spammed with calls", async () => {
         const NUMBER_OF_FILES = 10;
         const allFileNames: string[] = [];
         for (let i = 0; i < NUMBER_OF_FILES; i++) {

scripts/storage-emulator-integration/conformance/firebase.endpoints.test.ts

@@ -3,6 +3,7 @@ import { expect } from "chai";
 import * as admin from "firebase-admin";
 import * as fs from "fs";
 import * as supertest from "supertest";
+import { gunzipSync } from "zlib";
 import { TEST_ENV } from "./env";
 import { EmulatorEndToEndTest } from "../../integration-helpers/framework";
 import {
@@ -343,7 +344,6 @@ describe("Firebase Storage endpoint conformance tests", () => {
           })
           .expect(200)
           .then((res) => {
-            console.log(res);
             return new URL(res.header["x-goog-upload-url"]);
           });
 
@@ -475,6 +475,77 @@ describe("Firebase Storage endpoint conformance tests", () => {
     });
   });
 
+  describe("gzip", () => {
+    it("should serve gunzipped file by default", async () => {
+      const contents = Buffer.from("hello world");
+      const fileName = "gzippedFile";
+      const file = testBucket.file(fileName);
+      await file.save(contents, {
+        gzip: true,
+        contentType: "text/plain",
+      });
+
+      // Use requestClient since supertest will decompress the response body by default.
+      await new Promise((resolve, reject) => {
+        TEST_ENV.requestClient.get(
+          `${firebaseHost}/v0/b/${storageBucket}/o/${fileName}?alt=media`,
+          { headers: { ...authHeader } },
+          (res) => {
+            expect(res.headers["content-encoding"]).to.be.undefined;
+            expect(res.headers["content-length"]).to.be.undefined;
+            expect(res.headers["content-type"]).to.be.eql("text/plain");
+
+            let responseBody = Buffer.alloc(0);
+            res
+              .on("data", (chunk) => {
+                responseBody = Buffer.concat([responseBody, chunk]);
+              })
+              .on("end", () => {
+                expect(responseBody).to.be.eql(contents);
+              })
+              .on("close", resolve)
+              .on("error", reject);
+          }
+        );
+      });
+    });
+
+    it("should serve gzipped file if Accept-Encoding header allows", async () => {
+      const contents = Buffer.from("hello world");
+      const fileName = "gzippedFile";
+      const file = testBucket.file(fileName);
+      await file.save(contents, {
+        gzip: true,
+        contentType: "text/plain",
+      });
+
+      // Use requestClient since supertest will decompress the response body by default.
+      await new Promise((resolve, reject) => {
+        TEST_ENV.requestClient.get(
+          `${firebaseHost}/v0/b/${storageBucket}/o/${fileName}?alt=media`,
+          { headers: { ...authHeader, "Accept-Encoding": "gzip" } },
+          (res) => {
+            expect(res.headers["content-encoding"]).to.be.eql("gzip");
+            expect(res.headers["content-type"]).to.be.eql("text/plain");
+
+            let responseBody = Buffer.alloc(0);
+            res
+              .on("data", (chunk) => {
+                responseBody = Buffer.concat([responseBody, chunk]);
+              })
+              .on("end", () => {
+                expect(responseBody).to.not.be.eql(contents);
+                const decompressed = gunzipSync(responseBody);
+                expect(decompressed).to.be.eql(contents);
+              })
+              .on("close", resolve)
+              .on("error", reject);
+          }
+        );
+      });
+    });
+  });
+
   describe("tokens", () => {
     beforeEach(async () => {
       await testBucket.upload(smallFilePath, { destination: TEST_FILE_NAME });

scripts/storage-emulator-integration/conformance/gcs-js-sdk.test.ts

@@ -3,7 +3,6 @@ import { expect } from "chai";
 import * as admin from "firebase-admin";
 import * as fs from "fs";
 import { EmulatorEndToEndTest } from "../../integration-helpers/framework";
-import * as supertest from "supertest";
 import { TEST_ENV } from "./env";
 import {
   createRandomFile,
@@ -13,6 +12,7 @@ import {
   TEST_SETUP_TIMEOUT,
   getTmpDir,
 } from "../utils";
+import { gunzipSync } from "zlib";
 
 // Test case that should only run when targeting the emulator.
 // Example use: emulatorOnly.it("Local only test case", () => {...});
@@ -27,7 +27,6 @@ describe("GCS Javascript SDK conformance tests", () => {
   const storageBucket = TEST_ENV.appConfig.storageBucket;
   const otherStorageBucket = TEST_ENV.secondTestBucket;
   const storageHost = TEST_ENV.storageHost;
-  const firebaseHost = TEST_ENV.firebaseHost;
   const googleapisHost = TEST_ENV.googleapisHost;
 
   let test: EmulatorEndToEndTest;
@@ -109,14 +108,6 @@ describe("GCS Javascript SDK conformance tests", () => {
         fs.unlinkSync(content2);
       });
 
-      it("should handle gzip'd uploads", async () => {
-        // This appears to pass, but the file gets corrupted cause it's gzipped?
-        // expect(true).to.be.false;
-        await testBucket.upload(smallFilePath, {
-          gzip: true,
-        });
-      });
-
       it("should upload with provided metadata", async () => {
         const metadata = {
           contentDisposition: "attachment",
@@ -139,27 +130,15 @@ describe("GCS Javascript SDK conformance tests", () => {
           metadata: {},
         });
 
-        const cloudFile = testBucket.file(testFileName);
+        const file = testBucket.file(testFileName);
         const incomingMetadata = {
           metadata: {
             firebaseStorageDownloadTokens: "myFirstToken,mySecondToken",
           },
         };
-        await cloudFile.setMetadata(incomingMetadata);
-
-        // Check that the tokens are saved in Firebase metadata
-        await supertest(firebaseHost)
-          .get(`/v0/b/${testBucket.name}/o/${encodeURIComponent(testFileName)}`)
-          .expect(200)
-          .then((res) => {
-            const firebaseMd = res.body;
-            expect(firebaseMd.downloadTokens).to.equal(
-              incomingMetadata.metadata.firebaseStorageDownloadTokens
-            );
-          });
+        await file.setMetadata(incomingMetadata);
 
-        // Check that the tokens are saved in Cloud metadata
-        const [storedMetadata] = await cloudFile.getMetadata();
+        const [storedMetadata] = await file.getMetadata();
         expect(storedMetadata.metadata.firebaseStorageDownloadTokens).to.deep.equal(
           incomingMetadata.metadata.firebaseStorageDownloadTokens
         );
@@ -392,6 +371,26 @@ describe("GCS Javascript SDK conformance tests", () => {
   });
 
   describe(".file()", () => {
+    describe("#save()", () => {
+      it("should save", async () => {
+        const contents = Buffer.from("hello world");
+
+        const file = testBucket.file("gzippedFile");
+        await file.save(contents, { contentType: "text/plain" });
+
+        expect(file.metadata.contentType).to.be.eql("text/plain");
+        const [downloadedContents] = await file.download();
+        expect(downloadedContents).to.be.eql(contents);
+      });
+
+      it("should handle gzipped uploads", async () => {
+        const file = testBucket.file("gzippedFile");
+        await file.save("hello world", { gzip: true });
+
+        expect(file.metadata.contentEncoding).to.be.eql("gzip");
+      });
+    });
+
     describe("#exists()", () => {
       it("should return false for a file that does not exist", async () => {
         // Ensure that the file exists on the bucket before deleting it
@@ -488,6 +487,29 @@ describe("GCS Javascript SDK conformance tests", () => {
         expect(err).to.have.property("code", 404);
         expect(err).not.have.nested.property("errors[0]");
       });
+
+      it("should decompress gzipped file", async () => {
+        const contents = Buffer.from("hello world");
+
+        const file = testBucket.file("gzippedFile");
+        await file.save(contents, { gzip: true });
+
+        const [downloadedContents] = await file.download();
+        expect(downloadedContents).to.be.eql(contents);
+      });
+
+      it("should serve gzipped file if decompress option specified", async () => {
+        const contents = Buffer.from("hello world");
+
+        const file = testBucket.file("gzippedFile");
+        await file.save(contents, { gzip: true });
+
+        const [downloadedContents] = await file.download({ decompress: false });
+        expect(downloadedContents).to.not.be.eql(contents);
+
+        const ungzippedContents = gunzipSync(downloadedContents);
+        expect(ungzippedContents).to.be.eql(contents);
+      });
     });
 
     describe("#copy()", () => {

scripts/storage-emulator-integration/conformance/gcs.endpoints.test.ts

@@ -4,6 +4,7 @@ import * as admin from "firebase-admin";
 import * as fs from "fs";
 import * as supertest from "supertest";
 import { EmulatorEndToEndTest } from "../../integration-helpers/framework";
+import { gunzipSync } from "zlib";
 import { TEST_ENV } from "./env";
 import {
   EMULATORS_SHUTDOWN_DELAY_MS,
@@ -294,6 +295,77 @@ describe("GCS endpoint conformance tests", () => {
     });
   });
 
+  describe("Gzip", () => {
+    it("should serve gunzipped file by default", async () => {
+      const contents = Buffer.from("hello world");
+      const fileName = "gzippedFile";
+      const file = testBucket.file(fileName);
+      await file.save(contents, {
+        gzip: true,
+        contentType: "text/plain",
+      });
+
+      // Use requestClient since supertest will decompress the response body by default.
+      await new Promise((resolve, reject) => {
+        TEST_ENV.requestClient.get(
+          `${storageHost}/download/storage/v1/b/${storageBucket}/o/${fileName}?alt=media`,
+          { headers: { ...authHeader } },
+          (res) => {
+            expect(res.headers["content-encoding"]).to.be.undefined;
+            expect(res.headers["content-length"]).to.be.undefined;
+            expect(res.headers["content-type"]).to.be.eql("text/plain");
+
+            let responseBody = Buffer.alloc(0);
+            res
+              .on("data", (chunk) => {
+                responseBody = Buffer.concat([responseBody, chunk]);
+              })
+              .on("end", () => {
+                expect(responseBody).to.be.eql(contents);
+              })
+              .on("close", resolve)
+              .on("error", reject);
+          }
+        );
+      });
+    });
+
+    it("should serve gzipped file if Accept-Encoding header allows", async () => {
+      const contents = Buffer.from("hello world");
+      const fileName = "gzippedFile";
+      const file = testBucket.file(fileName);
+      await file.save(contents, {
+        gzip: true,
+        contentType: "text/plain",
+      });
+
+      // Use requestClient since supertest will decompress the response body by default.
+      await new Promise((resolve, reject) => {
+        TEST_ENV.requestClient.get(
+          `${storageHost}/download/storage/v1/b/${storageBucket}/o/${fileName}?alt=media`,
+          { headers: { ...authHeader, "Accept-Encoding": "gzip" } },
+          (res) => {
+            expect(res.headers["content-encoding"]).to.be.eql("gzip");
+            expect(res.headers["content-type"]).to.be.eql("text/plain");
+
+            let responseBody = Buffer.alloc(0);
+            res
+              .on("data", (chunk) => {
+                responseBody = Buffer.concat([responseBody, chunk]);
+              })
+              .on("end", () => {
+                expect(responseBody).to.not.be.eql(contents);
+                const decompressed = gunzipSync(responseBody);
+                expect(decompressed).to.be.eql(contents);
+              })
+              .on("close", resolve)
+              .on("error", reject);
+          }
+        );
+      });
+    });
+  });
+
   describe("List protocols", () => {
     describe("list objects", () => {
       // This test is for the '/storage/v1/b/:bucketId/o' url pattern, which is used specifically by the GO Admin SDK