diff --git a/experimental/puppeteer-firefox/lib/NetworkManager.js b/experimental/puppeteer-firefox/lib/NetworkManager.js index 1999754564694..d35676c309352 100644 --- a/experimental/puppeteer-firefox/lib/NetworkManager.js +++ b/experimental/puppeteer-firefox/lib/NetworkManager.js @@ -60,7 +60,7 @@ class NetworkManager extends EventEmitter { const request = this._requests.get(event.requestId); if (!request) return; - const response = new Response(request, event); + const response = new Response(this._session, request, event); request._response = response; this.emit(Events.NetworkManager.Response, response); } @@ -71,8 +71,12 @@ class NetworkManager extends EventEmitter { return; // Keep redirected requests in the map for future reference in redirectChain. const isRedirected = request.response().status() >= 300 && request.response().status() <= 399; - if (!isRedirected) + if (isRedirected) { + request.response()._bodyLoadedPromiseFulfill.call(null, new Error('Response body is unavailable for redirect responses')); + } else { this._requests.delete(request._id); + request.response()._bodyLoadedPromiseFulfill.call(null); + } this.emit(Events.NetworkManager.RequestFinished, request); } @@ -81,6 +85,8 @@ class NetworkManager extends EventEmitter { if (!request) return; this._requests.delete(request._id); + if (request.response()) + request.response()._bodyLoadedPromiseFulfill.call(null); request._errorText = event.errorCode; this.emit(Events.NetworkManager.RequestFailed, request); } @@ -200,7 +206,8 @@ class Request { } class Response { - constructor(request, payload) { + constructor(session, request, payload) { + this._session = session; this._request = request; this._remoteIPAddress = payload.remoteIPAddress; this._remotePort = payload.remotePort; @@ -210,6 +217,44 @@ class Response { this._securityDetails = payload.securityDetails ? new SecurityDetails(payload.securityDetails) : null; for (const {name, value} of payload.headers) this._headers[name.toLowerCase()] = value; + this._bodyLoadedPromise = new Promise(fulfill => { + this._bodyLoadedPromiseFulfill = fulfill; + }); + } + + /** + * @return {!Promise} + */ + buffer() { + if (!this._contentPromise) { + this._contentPromise = this._bodyLoadedPromise.then(async error => { + if (error) + throw error; + const response = await this._session.send('Network.getResponseBody', { + requestId: this._request._id + }); + if (response.evicted) + throw new Error(`Response body for ${this._request.method()} ${this._request.url()} was evicted!`); + return Buffer.from(response.base64body, 'base64'); + }); + } + return this._contentPromise; + } + + /** + * @return {!Promise} + */ + async text() { + const content = await this.buffer(); + return content.toString('utf8'); + } + + /** + * @return {!Promise} + */ + async json() { + const content = await this.text(); + return JSON.parse(content); } securityDetails() { diff --git a/experimental/puppeteer-firefox/misc/puppeteer.cfg b/experimental/puppeteer-firefox/misc/puppeteer.cfg index d902041c44598..edf2d504e7afc 100644 --- a/experimental/puppeteer-firefox/misc/puppeteer.cfg +++ b/experimental/puppeteer-firefox/misc/puppeteer.cfg @@ -17,6 +17,12 @@ pref("browser.newtabpage.enabled", false); // Disable topstories pref("browser.newtabpage.activity-stream.feeds.section.topstories", false); +// DevTools JSONViewer sometimes fails to load dependencies with its require.js. +// This doesn't affect Puppeteer operations, but spams console with a lot of +// unpleasant errors. +// (bug 1424372) +pref("devtools.jsonview.enabled", false); + // Increase the APZ content response timeout in tests to 1 minute. // This is to accommodate the fact that test environments tends to be // slower than production environments (with the b2g emulator being diff --git a/experimental/puppeteer-firefox/package.json b/experimental/puppeteer-firefox/package.json index af11aa02168ab..7acdf661ec20b 100644 --- a/experimental/puppeteer-firefox/package.json +++ b/experimental/puppeteer-firefox/package.json @@ -9,7 +9,7 @@ "node": ">=8.9.4" }, "puppeteer": { - "firefox_revision": "f7b25713dd00f0deda7032dc25a72d4c7b42446e" + "firefox_revision": "3ba79216e3c5ae4e85006047cdd93eac4197427d" }, "scripts": { "install": "node install.js", diff --git a/test/navigation.spec.js b/test/navigation.spec.js index 66905a486b79f..04bfebd46b5fa 100644 --- a/test/navigation.spec.js +++ b/test/navigation.spec.js @@ -499,7 +499,7 @@ module.exports.addTests = function({testRunner, expect, Errors, CHROME}) { const error = await navigationPromise; expect(error.message).toBe('Navigating frame was detached'); }); - it_fails_ffox('should return matching responses', async({page, server}) => { + it('should return matching responses', async({page, server}) => { // Disable cache: otherwise, chromium will cache similar requests. await page.setCacheEnabled(false); await page.goto(server.EMPTY_PAGE); diff --git a/test/network.spec.js b/test/network.spec.js index d024d3ce4b788..469c8404c43e4 100644 --- a/test/network.spec.js +++ b/test/network.spec.js @@ -156,52 +156,18 @@ module.exports.addTests = function({testRunner, expect, CHROME}) { }); }); - describe('Network Events', function() { - it('Page.Events.Request', async({page, server}) => { - const requests = []; - page.on('request', request => requests.push(request)); - await page.goto(server.EMPTY_PAGE); - expect(requests.length).toBe(1); - expect(requests[0].url()).toBe(server.EMPTY_PAGE); - expect(requests[0].resourceType()).toBe('document'); - expect(requests[0].method()).toBe('GET'); - expect(requests[0].response()).toBeTruthy(); - expect(requests[0].frame() === page.mainFrame()).toBe(true); - expect(requests[0].frame().url()).toBe(server.EMPTY_PAGE); - }); - it('Page.Events.Response', async({page, server}) => { - const responses = []; - page.on('response', response => responses.push(response)); - await page.goto(server.EMPTY_PAGE); - expect(responses.length).toBe(1); - expect(responses[0].url()).toBe(server.EMPTY_PAGE); - expect(responses[0].status()).toBe(200); - expect(responses[0].ok()).toBe(true); - expect(responses[0].request()).toBeTruthy(); - const remoteAddress = responses[0].remoteAddress(); - // Either IPv6 or IPv4, depending on environment. - expect(remoteAddress.ip.includes('::1') || remoteAddress.ip === '127.0.0.1').toBe(true); - expect(remoteAddress.port).toBe(server.PORT); - }); - - it('Response.statusText', async({page, server}) => { - server.setRoute('/cool', (req, res) => { - res.writeHead(200, 'cool!'); - res.end(); - }); - const response = await page.goto(server.PREFIX + '/cool'); - expect(response.statusText()).toBe('cool!'); + describe('Response.text', function() { + it('should work', async({page, server}) => { + const response = await page.goto(server.PREFIX + '/simple.json'); + expect(await response.text()).toBe('{"foo": "bar"}\n'); }); - - it_fails_ffox('Page.Events.Response should provide body', async({page, server}) => { - let response = null; - page.on('response', r => response = r); - await page.goto(server.PREFIX + '/simple.json'); - expect(response).toBeTruthy(); + it('should return uncompressed text', async({page, server}) => { + server.enableGzip('/simple.json'); + const response = await page.goto(server.PREFIX + '/simple.json'); + expect(response.headers()['content-encoding']).toBe('gzip'); expect(await response.text()).toBe('{"foo": "bar"}\n'); - expect(await response.json()).toEqual({foo: 'bar'}); }); - it_fails_ffox('Page.Events.Response should throw when requesting body of redirected response', async({page, server}) => { + it('should throw when requesting body of redirected response', async({page, server}) => { server.setRedirect('/foo.html', '/empty.html'); const response = await page.goto(server.PREFIX + '/foo.html'); const redirectChain = response.request().redirectChain(); @@ -212,23 +178,25 @@ module.exports.addTests = function({testRunner, expect, CHROME}) { await redirected.text().catch(e => error = e); expect(error.message).toContain('Response body is unavailable for redirect responses'); }); - it_fails_ffox('Page.Events.Response should not report body unless request is finished', async({page, server}) => { + it('should wait until response completes', async({page, server}) => { await page.goto(server.EMPTY_PAGE); // Setup server to trap request. let serverResponse = null; server.setRoute('/get', (req, res) => { serverResponse = res; + // In Firefox, |fetch| will be hanging until it receives |Content-Type| header + // from server. + res.setHeader('Content-Type', 'text/plain; charset=utf-8'); res.write('hello '); }); // Setup page to trap response. - let pageResponse = null; let requestFinished = false; - page.on('response', r => pageResponse = r); - page.on('requestfinished', () => requestFinished = true); + page.on('requestfinished', r => requestFinished = requestFinished || r.url().includes('/get')); // send request and wait for server response - await Promise.all([ + const [pageResponse] = await Promise.all([ + page.waitForResponse(r => !utils.isFavicon(r.request())), page.evaluate(() => fetch('./get', { method: 'GET'})), - utils.waitEvent(page, 'response') + server.waitForRequest('/get'), ]); expect(serverResponse).toBeTruthy(); @@ -243,6 +211,68 @@ module.exports.addTests = function({testRunner, expect, CHROME}) { await new Promise(x => serverResponse.end('ld!', x)); expect(await responseText).toBe('hello world!'); }); + }); + + describe('Response.json', function() { + it('should work', async({page, server}) => { + const response = await page.goto(server.PREFIX + '/simple.json'); + expect(await response.json()).toEqual({foo: 'bar'}); + }); + }); + + describe('Response.buffer', function() { + it('should work', async({page, server}) => { + const response = await page.goto(server.PREFIX + '/pptr.png'); + const imageBuffer = fs.readFileSync(path.join(__dirname, 'assets', 'pptr.png')); + const responseBuffer = await response.buffer(); + expect(responseBuffer.equals(imageBuffer)).toBe(true); + }); + it('should work with compression', async({page, server}) => { + server.enableGzip('/pptr.png'); + const response = await page.goto(server.PREFIX + '/pptr.png'); + const imageBuffer = fs.readFileSync(path.join(__dirname, 'assets', 'pptr.png')); + const responseBuffer = await response.buffer(); + expect(responseBuffer.equals(imageBuffer)).toBe(true); + }); + }); + + describe('Network Events', function() { + it('Page.Events.Request', async({page, server}) => { + const requests = []; + page.on('request', request => requests.push(request)); + await page.goto(server.EMPTY_PAGE); + expect(requests.length).toBe(1); + expect(requests[0].url()).toBe(server.EMPTY_PAGE); + expect(requests[0].resourceType()).toBe('document'); + expect(requests[0].method()).toBe('GET'); + expect(requests[0].response()).toBeTruthy(); + expect(requests[0].frame() === page.mainFrame()).toBe(true); + expect(requests[0].frame().url()).toBe(server.EMPTY_PAGE); + }); + it('Page.Events.Response', async({page, server}) => { + const responses = []; + page.on('response', response => responses.push(response)); + await page.goto(server.EMPTY_PAGE); + expect(responses.length).toBe(1); + expect(responses[0].url()).toBe(server.EMPTY_PAGE); + expect(responses[0].status()).toBe(200); + expect(responses[0].ok()).toBe(true); + expect(responses[0].request()).toBeTruthy(); + const remoteAddress = responses[0].remoteAddress(); + // Either IPv6 or IPv4, depending on environment. + expect(remoteAddress.ip.includes('::1') || remoteAddress.ip === '127.0.0.1').toBe(true); + expect(remoteAddress.port).toBe(server.PORT); + }); + + it('Response.statusText', async({page, server}) => { + server.setRoute('/cool', (req, res) => { + res.writeHead(200, 'cool!'); + res.end(); + }); + const response = await page.goto(server.PREFIX + '/cool'); + expect(response.statusText()).toBe('cool!'); + }); + it('Page.Events.RequestFailed', async({page, server}) => { await page.setRequestInterception(true); page.on('request', request => { diff --git a/utils/testserver/index.js b/utils/testserver/index.js index a0bbd7adc54de..54b102134dcf8 100644 --- a/utils/testserver/index.js +++ b/utils/testserver/index.js @@ -80,6 +80,8 @@ class TestServer { this._auths = new Map(); /** @type {!Map} */ this._csp = new Map(); + /** @type {!Set} */ + this._gzipRoutes = new Set(); /** @type {!Map} */ this._requestSubscribers = new Map(); } @@ -111,6 +113,10 @@ class TestServer { this._auths.set(path, {username, password}); } + enableGzip(path) { + this._gzipRoutes.add(path); + } + /** * @param {string} path * @param {string} csp @@ -169,6 +175,7 @@ class TestServer { this._routes.clear(); this._auths.clear(); this._csp.clear(); + this._gzipRoutes.clear(); const error = new Error('Static Server has been reset'); for (const subscriber of this._requestSubscribers.values()) subscriber[rejectSymbol].call(null, error); @@ -230,14 +237,22 @@ class TestServer { if (this._csp.has(pathName)) response.setHeader('Content-Security-Policy', this._csp.get(pathName)); - fs.readFile(filePath, function(err, data) { + fs.readFile(filePath, (err, data) => { if (err) { response.statusCode = 404; response.end(`File not found: ${filePath}`); return; } response.setHeader('Content-Type', mime.getType(filePath)); - response.end(data); + if (this._gzipRoutes.has(pathName)) { + response.setHeader('Content-Encoding', 'gzip'); + const zlib = require('zlib'); + zlib.gzip(data, (_, result) => { + response.end(result); + }); + } else { + response.end(data); + } }); }