feat: support Response.buffer(), Response.json() and Response.text() (p…

…uppeteer#4063)

This patch:
- implements Response.buffer() and other methods
- splits out relevant tests into a separate test suites
- implements `testServer.enableGzip()` method to optionally gzip
  certain routes in tests
- adds tests to make sure `Response.text()` returns expected results
  for binary and compressed responses.

experimental/puppeteer-firefox/lib/NetworkManager.js

@@ -60,7 +60,7 @@ class NetworkManager extends EventEmitter {
     const request = this._requests.get(event.requestId);
     if (!request)
       return;
-    const response = new Response(request, event);
+    const response = new Response(this._session, request, event);
     request._response = response;
     this.emit(Events.NetworkManager.Response, response);
   }
@@ -71,8 +71,12 @@ class NetworkManager extends EventEmitter {
       return;
     // Keep redirected requests in the map for future reference in redirectChain.
     const isRedirected = request.response().status() >= 300 && request.response().status() <= 399;
-    if (!isRedirected)
+    if (isRedirected) {
+      request.response()._bodyLoadedPromiseFulfill.call(null, new Error('Response body is unavailable for redirect responses'));
+    } else {
       this._requests.delete(request._id);
+      request.response()._bodyLoadedPromiseFulfill.call(null);
+    }
     this.emit(Events.NetworkManager.RequestFinished, request);
   }
 
@@ -81,6 +85,8 @@ class NetworkManager extends EventEmitter {
     if (!request)
       return;
     this._requests.delete(request._id);
+    if (request.response())
+      request.response()._bodyLoadedPromiseFulfill.call(null);
     request._errorText = event.errorCode;
     this.emit(Events.NetworkManager.RequestFailed, request);
   }
@@ -200,7 +206,8 @@ class Request {
 }
 
 class Response {
-  constructor(request, payload) {
+  constructor(session, request, payload) {
+    this._session = session;
     this._request = request;
     this._remoteIPAddress = payload.remoteIPAddress;
     this._remotePort = payload.remotePort;
@@ -210,6 +217,44 @@ class Response {
     this._securityDetails = payload.securityDetails ? new SecurityDetails(payload.securityDetails) : null;
     for (const {name, value} of payload.headers)
       this._headers[name.toLowerCase()] = value;
+    this._bodyLoadedPromise = new Promise(fulfill => {
+      this._bodyLoadedPromiseFulfill = fulfill;
+    });
+  }
+
+  /**
+   * @return {!Promise<!Buffer>}
+   */
+  buffer() {
+    if (!this._contentPromise) {
+      this._contentPromise = this._bodyLoadedPromise.then(async error => {
+        if (error)
+          throw error;
+        const response = await this._session.send('Network.getResponseBody', {
+          requestId: this._request._id
+        });
+        if (response.evicted)
+          throw new Error(`Response body for ${this._request.method()} ${this._request.url()} was evicted!`);
+        return Buffer.from(response.base64body, 'base64');
+      });
+    }
+    return this._contentPromise;
+  }
+
+  /**
+   * @return {!Promise<string>}
+   */
+  async text() {
+    const content = await this.buffer();
+    return content.toString('utf8');
+  }
+
+  /**
+   * @return {!Promise<!Object>}
+   */
+  async json() {
+    const content = await this.text();
+    return JSON.parse(content);
   }
 
   securityDetails() {

experimental/puppeteer-firefox/misc/puppeteer.cfg

@@ -17,6 +17,12 @@ pref("browser.newtabpage.enabled", false);
 // Disable topstories
 pref("browser.newtabpage.activity-stream.feeds.section.topstories", false);
 
+// DevTools JSONViewer sometimes fails to load dependencies with its require.js.
+// This doesn't affect Puppeteer operations, but spams console with a lot of
+// unpleasant errors.
+// (bug 1424372)
+pref("devtools.jsonview.enabled", false);
+
 // Increase the APZ content response timeout in tests to 1 minute.
 // This is to accommodate the fact that test environments tends to be
 // slower than production environments (with the b2g emulator being

experimental/puppeteer-firefox/package.json

@@ -9,7 +9,7 @@
     "node": ">=8.9.4"
   },
   "puppeteer": {
-    "firefox_revision": "f7b25713dd00f0deda7032dc25a72d4c7b42446e"
+    "firefox_revision": "3ba79216e3c5ae4e85006047cdd93eac4197427d"
   },
   "scripts": {
     "install": "node install.js",

test/navigation.spec.js

@@ -499,7 +499,7 @@ module.exports.addTests = function({testRunner, expect, Errors, CHROME}) {
       const error = await navigationPromise;
       expect(error.message).toBe('Navigating frame was detached');
     });
-    it_fails_ffox('should return matching responses', async({page, server}) => {
+    it('should return matching responses', async({page, server}) => {
       // Disable cache: otherwise, chromium will cache similar requests.
       await page.setCacheEnabled(false);
       await page.goto(server.EMPTY_PAGE);

test/network.spec.js

@@ -156,52 +156,18 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
     });
   });
 
-  describe('Network Events', function() {
-    it('Page.Events.Request', async({page, server}) => {
-      const requests = [];
-      page.on('request', request => requests.push(request));
-      await page.goto(server.EMPTY_PAGE);
-      expect(requests.length).toBe(1);
-      expect(requests[0].url()).toBe(server.EMPTY_PAGE);
-      expect(requests[0].resourceType()).toBe('document');
-      expect(requests[0].method()).toBe('GET');
-      expect(requests[0].response()).toBeTruthy();
-      expect(requests[0].frame() === page.mainFrame()).toBe(true);
-      expect(requests[0].frame().url()).toBe(server.EMPTY_PAGE);
-    });
-    it('Page.Events.Response', async({page, server}) => {
-      const responses = [];
-      page.on('response', response => responses.push(response));
-      await page.goto(server.EMPTY_PAGE);
-      expect(responses.length).toBe(1);
-      expect(responses[0].url()).toBe(server.EMPTY_PAGE);
-      expect(responses[0].status()).toBe(200);
-      expect(responses[0].ok()).toBe(true);
-      expect(responses[0].request()).toBeTruthy();
-      const remoteAddress = responses[0].remoteAddress();
-      // Either IPv6 or IPv4, depending on environment.
-      expect(remoteAddress.ip.includes('::1') || remoteAddress.ip === '127.0.0.1').toBe(true);
-      expect(remoteAddress.port).toBe(server.PORT);
-    });
-
-    it('Response.statusText', async({page, server}) => {
-      server.setRoute('/cool', (req, res) => {
-        res.writeHead(200, 'cool!');
-        res.end();
-      });
-      const response = await page.goto(server.PREFIX + '/cool');
-      expect(response.statusText()).toBe('cool!');
+  describe('Response.text', function() {
+    it('should work', async({page, server}) => {
+      const response = await page.goto(server.PREFIX + '/simple.json');
+      expect(await response.text()).toBe('{"foo": "bar"}\n');
     });
-
-    it_fails_ffox('Page.Events.Response should provide body', async({page, server}) => {
-      let response = null;
-      page.on('response', r => response = r);
-      await page.goto(server.PREFIX + '/simple.json');
-      expect(response).toBeTruthy();
+    it('should return uncompressed text', async({page, server}) => {
+      server.enableGzip('/simple.json');
+      const response = await page.goto(server.PREFIX + '/simple.json');
+      expect(response.headers()['content-encoding']).toBe('gzip');
       expect(await response.text()).toBe('{"foo": "bar"}\n');
-      expect(await response.json()).toEqual({foo: 'bar'});
     });
-    it_fails_ffox('Page.Events.Response should throw when requesting body of redirected response', async({page, server}) => {
+    it('should throw when requesting body of redirected response', async({page, server}) => {
       server.setRedirect('/foo.html', '/empty.html');
       const response = await page.goto(server.PREFIX + '/foo.html');
       const redirectChain = response.request().redirectChain();
@@ -212,23 +178,25 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
       await redirected.text().catch(e => error = e);
       expect(error.message).toContain('Response body is unavailable for redirect responses');
     });
-    it_fails_ffox('Page.Events.Response should not report body unless request is finished', async({page, server}) => {
+    it('should wait until response completes', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       // Setup server to trap request.
       let serverResponse = null;
       server.setRoute('/get', (req, res) => {
         serverResponse = res;
+        // In Firefox, |fetch| will be hanging until it receives |Content-Type| header
+        // from server.
+        res.setHeader('Content-Type', 'text/plain; charset=utf-8');
         res.write('hello ');
       });
       // Setup page to trap response.
-      let pageResponse = null;
       let requestFinished = false;
-      page.on('response', r => pageResponse = r);
-      page.on('requestfinished', () => requestFinished = true);
+      page.on('requestfinished', r => requestFinished = requestFinished || r.url().includes('/get'));
       // send request and wait for server response
-      await Promise.all([
+      const [pageResponse] = await Promise.all([
+        page.waitForResponse(r => !utils.isFavicon(r.request())),
         page.evaluate(() => fetch('./get', { method: 'GET'})),
-        utils.waitEvent(page, 'response')
+        server.waitForRequest('/get'),
       ]);
 
       expect(serverResponse).toBeTruthy();
@@ -243,6 +211,68 @@ module.exports.addTests = function({testRunner, expect, CHROME}) {
       await new Promise(x => serverResponse.end('ld!', x));
       expect(await responseText).toBe('hello world!');
     });
+  });
+
+  describe('Response.json', function() {
+    it('should work', async({page, server}) => {
+      const response = await page.goto(server.PREFIX + '/simple.json');
+      expect(await response.json()).toEqual({foo: 'bar'});
+    });
+  });
+
+  describe('Response.buffer', function() {
+    it('should work', async({page, server}) => {
+      const response = await page.goto(server.PREFIX + '/pptr.png');
+      const imageBuffer = fs.readFileSync(path.join(__dirname, 'assets', 'pptr.png'));
+      const responseBuffer = await response.buffer();
+      expect(responseBuffer.equals(imageBuffer)).toBe(true);
+    });
+    it('should work with compression', async({page, server}) => {
+      server.enableGzip('/pptr.png');
+      const response = await page.goto(server.PREFIX + '/pptr.png');
+      const imageBuffer = fs.readFileSync(path.join(__dirname, 'assets', 'pptr.png'));
+      const responseBuffer = await response.buffer();
+      expect(responseBuffer.equals(imageBuffer)).toBe(true);
+    });
+  });
+
+  describe('Network Events', function() {
+    it('Page.Events.Request', async({page, server}) => {
+      const requests = [];
+      page.on('request', request => requests.push(request));
+      await page.goto(server.EMPTY_PAGE);
+      expect(requests.length).toBe(1);
+      expect(requests[0].url()).toBe(server.EMPTY_PAGE);
+      expect(requests[0].resourceType()).toBe('document');
+      expect(requests[0].method()).toBe('GET');
+      expect(requests[0].response()).toBeTruthy();
+      expect(requests[0].frame() === page.mainFrame()).toBe(true);
+      expect(requests[0].frame().url()).toBe(server.EMPTY_PAGE);
+    });
+    it('Page.Events.Response', async({page, server}) => {
+      const responses = [];
+      page.on('response', response => responses.push(response));
+      await page.goto(server.EMPTY_PAGE);
+      expect(responses.length).toBe(1);
+      expect(responses[0].url()).toBe(server.EMPTY_PAGE);
+      expect(responses[0].status()).toBe(200);
+      expect(responses[0].ok()).toBe(true);
+      expect(responses[0].request()).toBeTruthy();
+      const remoteAddress = responses[0].remoteAddress();
+      // Either IPv6 or IPv4, depending on environment.
+      expect(remoteAddress.ip.includes('::1') || remoteAddress.ip === '127.0.0.1').toBe(true);
+      expect(remoteAddress.port).toBe(server.PORT);
+    });
+
+    it('Response.statusText', async({page, server}) => {
+      server.setRoute('/cool', (req, res) => {
+        res.writeHead(200, 'cool!');
+        res.end();
+      });
+      const response = await page.goto(server.PREFIX + '/cool');
+      expect(response.statusText()).toBe('cool!');
+    });
+
     it('Page.Events.RequestFailed', async({page, server}) => {
       await page.setRequestInterception(true);
       page.on('request', request => {

utils/testserver/index.js

@@ -80,6 +80,8 @@ class TestServer {
     this._auths = new Map();
     /** @type {!Map<string, string>} */
     this._csp = new Map();
+    /** @type {!Set<string>} */
+    this._gzipRoutes = new Set();
     /** @type {!Map<string, !Promise>} */
     this._requestSubscribers = new Map();
   }
@@ -111,6 +113,10 @@ class TestServer {
     this._auths.set(path, {username, password});
   }
 
+  enableGzip(path) {
+    this._gzipRoutes.add(path);
+  }
+
   /**
    * @param {string} path
    * @param {string} csp
@@ -169,6 +175,7 @@ class TestServer {
     this._routes.clear();
     this._auths.clear();
     this._csp.clear();
+    this._gzipRoutes.clear();
     const error = new Error('Static Server has been reset');
     for (const subscriber of this._requestSubscribers.values())
       subscriber[rejectSymbol].call(null, error);
@@ -230,14 +237,22 @@ class TestServer {
     if (this._csp.has(pathName))
       response.setHeader('Content-Security-Policy', this._csp.get(pathName));
 
-    fs.readFile(filePath, function(err, data) {
+    fs.readFile(filePath, (err, data) => {
       if (err) {
         response.statusCode = 404;
         response.end(`File not found: ${filePath}`);
         return;
       }
       response.setHeader('Content-Type', mime.getType(filePath));
-      response.end(data);
+      if (this._gzipRoutes.has(pathName)) {
+        response.setHeader('Content-Encoding', 'gzip');
+        const zlib = require('zlib');
+        zlib.gzip(data, (_, result) => {
+          response.end(result);
+        });
+      } else {
+        response.end(data);
+      }
     });
   }