From 14fb3e38db7c97fc3ea76c65e6f219b0ddb3b54f Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Tue, 5 Feb 2019 22:32:41 -0800 Subject: [PATCH] chore(firefox): mark all Puppeteer tests that are failing on FF (#3924) This patch: - introduces new testRunner methods `addTestDSL` and `addSuiteDSL` to add annotated test / suite. - introduces new test/suite declaration methods: `it_fails_ffox` and `describe_fails_ffox`. These are equal to `it`/`describe` for chromium tests and to `xit`/`xdescribe` for firefox. - marks all unsupported tests with `it_fails_ffox` - adds a new command-line flag `'--firefox-status'` to `//test/test.js`. This flag dumps current amount of tests that are intentionally skipped for Firefox. End goal: get rid of all `it_fails_ffox` and `describe_fails_ffox` tests. Drive-By: remove cookie tests "afterEach" hook that was removing cookies - it's not needed any more since every test is run in a designated browser context. References #3889 --- test/accessibility.spec.js | 4 +- test/browser.spec.js | 8 ++-- test/browsercontext.spec.js | 4 +- test/click.spec.js | 6 +-- test/cookies.spec.js | 9 +--- test/elementhandle.spec.js | 8 ++-- test/emulation.spec.js | 10 ++--- test/evaluation.spec.js | 20 ++++----- test/frame.spec.js | 8 ++-- test/ignorehttpserrors.spec.js | 8 ++-- test/input.spec.js | 6 +-- test/jshandle.spec.js | 10 ++--- test/keyboard.spec.js | 14 +++--- test/launcher.spec.js | 36 ++++++++-------- test/mouse.spec.js | 10 ++--- test/navigation.spec.js | 78 +++++++++++++++++----------------- test/network.spec.js | 16 +++---- test/page.spec.js | 62 +++++++++++++-------------- test/puppeteer.spec.js | 12 +++--- test/screenshot.spec.js | 6 +-- test/target.spec.js | 14 +++--- test/test.js | 23 ++++++++++ test/touchscreen.spec.js | 4 +- test/waittask.spec.js | 18 ++++---- test/worker.spec.js | 4 +- utils/testrunner/TestRunner.js | 34 +++++++++------ 26 files changed, 230 insertions(+), 202 deletions(-) diff --git a/test/accessibility.spec.js b/test/accessibility.spec.js index a9eb88a4bce2b..1a3db0b241f67 100644 --- a/test/accessibility.spec.js +++ b/test/accessibility.spec.js @@ -15,11 +15,11 @@ */ module.exports.addTests = function({testRunner, expect}) { - const {describe, xdescribe, fdescribe} = testRunner; + const {describe, xdescribe, fdescribe, describe_fails_ffox} = testRunner; const {it, fit, xit} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; - describe('Accessibility', function() { + describe_fails_ffox('Accessibility', function() { it('should work', async function({page}) { await page.setContent(` diff --git a/test/browser.spec.js b/test/browser.spec.js index 631c3cd306944..5f373f70d6bf8 100644 --- a/test/browser.spec.js +++ b/test/browser.spec.js @@ -14,13 +14,13 @@ * limitations under the License. */ -module.exports.addTests = function({testRunner, expect, headless, puppeteer, FFOX}) { +module.exports.addTests = function({testRunner, expect, headless, puppeteer}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('Browser.target', function() { - it('should return browser target', async({browser}) => { + it_fails_ffox('should return browser target', async({browser}) => { const target = browser.target(); expect(target.type()).toBe('browser'); }); @@ -31,7 +31,7 @@ module.exports.addTests = function({testRunner, expect, headless, puppeteer, FFO const process = await browser.process(); expect(process.pid).toBeGreaterThan(0); }); - (FFOX ? xit : it)('should not return child_process for remote browser', async function({browser}) { + it_fails_ffox('should not return child_process for remote browser', async function({browser}) { const browserWSEndpoint = browser.wsEndpoint(); const remoteBrowser = await puppeteer.connect({browserWSEndpoint}); expect(remoteBrowser.process()).toBe(null); diff --git a/test/browsercontext.spec.js b/test/browsercontext.spec.js index 54ed4afab0095..0accba66d7283 100644 --- a/test/browsercontext.spec.js +++ b/test/browsercontext.spec.js @@ -18,7 +18,7 @@ const utils = require('./utils'); module.exports.addTests = function({testRunner, expect, puppeteer, Errors}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; const {TimeoutError} = Errors; @@ -141,7 +141,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer, Errors}) { ]); expect(browser.browserContexts().length).toBe(1); }); - it('should work across sessions', async function({browser, server}) { + it_fails_ffox('should work across sessions', async function({browser, server}) { expect(browser.browserContexts().length).toBe(1); const context = await browser.createIncognitoBrowserContext(); expect(browser.browserContexts().length).toBe(2); diff --git a/test/click.spec.js b/test/click.spec.js index 945e068ba32c5..d9995a7d3b7a7 100644 --- a/test/click.spec.js +++ b/test/click.spec.js @@ -20,7 +20,7 @@ const iPhone = DeviceDescriptors['iPhone 6']; module.exports.addTests = function({testRunner, expect}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('Page.click', function() { it('should click the button', async({page, server}) => { @@ -28,7 +28,7 @@ module.exports.addTests = function({testRunner, expect}) { await page.click('button'); expect(await page.evaluate(() => result)).toBe('Clicked'); }); - it('should click the button if window.Node is removed', async({page, server}) => { + it_fails_ffox('should click the button if window.Node is removed', async({page, server}) => { await page.goto(server.PREFIX + '/input/button.html'); await page.evaluate(() => delete window.Node); await page.click('button'); @@ -41,7 +41,7 @@ module.exports.addTests = function({testRunner, expect}) { await page.click('button'); expect(await page.evaluate(() => result)).toBe('Clicked'); }); - it('should click with disabled javascript', async({page, server}) => { + it_fails_ffox('should click with disabled javascript', async({page, server}) => { await page.setJavaScriptEnabled(false); await page.goto(server.PREFIX + '/wrappedlink.html'); await Promise.all([ diff --git a/test/cookies.spec.js b/test/cookies.spec.js index a4b1ecc3ff7ee..5148162a63a03 100644 --- a/test/cookies.spec.js +++ b/test/cookies.spec.js @@ -15,16 +15,11 @@ */ module.exports.addTests = function({testRunner, expect}) { - const {describe, xdescribe, fdescribe} = testRunner; + const {describe, xdescribe, fdescribe, describe_fails_ffox} = testRunner; const {it, fit, xit} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; - describe('Cookies', function() { - afterEach(async({page, server}) => { - const cookies = await page.cookies(server.PREFIX + '/grid.html', server.CROSS_PROCESS_PREFIX); - for (const cookie of cookies) - await page.deleteCookie(cookie); - }); + describe_fails_ffox('Cookies', function() { it('should set and get cookies', async({page, server}) => { await page.goto(server.PREFIX + '/grid.html'); expect(await page.cookies()).toEqual([]); diff --git a/test/elementhandle.spec.js b/test/elementhandle.spec.js index d92d6df0dffe4..39e783eab9f54 100644 --- a/test/elementhandle.spec.js +++ b/test/elementhandle.spec.js @@ -17,8 +17,8 @@ const utils = require('./utils'); module.exports.addTests = function({testRunner, expect}) { - const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {describe, xdescribe, fdescribe, describe_fails_ffox} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('ElementHandle.boundingBox', function() { @@ -29,7 +29,7 @@ module.exports.addTests = function({testRunner, expect}) { const box = await elementHandle.boundingBox(); expect(box).toEqual({ x: 100, y: 50, width: 50, height: 50 }); }); - it('should handle nested frames', async({page, server}) => { + it_fails_ffox('should handle nested frames', async({page, server}) => { await page.setViewport({width: 500, height: 500}); await page.goto(server.PREFIX + '/frames/nested-frames.html'); const nestedFrame = page.frames()[1].childFrames()[1]; @@ -66,7 +66,7 @@ module.exports.addTests = function({testRunner, expect}) { }); }); - describe('ElementHandle.boxModel', function() { + describe_fails_ffox('ElementHandle.boxModel', function() { it('should work', async({page, server}) => { await page.goto(server.PREFIX + '/resetcss.html'); diff --git a/test/emulation.spec.js b/test/emulation.spec.js index 1ee318cf2ed1e..71b4704af85a0 100644 --- a/test/emulation.spec.js +++ b/test/emulation.spec.js @@ -20,8 +20,8 @@ const iPhone = DeviceDescriptors['iPhone 6']; const iPhoneLandscape = DeviceDescriptors['iPhone 6 landscape']; module.exports.addTests = function({testRunner, expect, product}) { - const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {describe, xdescribe, fdescribe, describe_fails_ffox} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('Page.viewport', function() { @@ -72,7 +72,7 @@ module.exports.addTests = function({testRunner, expect, product}) { await page.addScriptTag({url: server.PREFIX + '/modernizr.js'}); expect(await page.evaluate(() => Modernizr.touchevents)).toBe(true); }); - it('should support landscape emulation', async({page, server}) => { + it_fails_ffox('should support landscape emulation', async({page, server}) => { await page.goto(server.PREFIX + '/mobile.html'); expect(await page.evaluate(() => screen.orientation.type)).toBe('portrait-primary'); await page.setViewport(iPhoneLandscape.viewport); @@ -82,7 +82,7 @@ module.exports.addTests = function({testRunner, expect, product}) { }); }); - describe('Page.emulate', function() { + describe_fails_ffox('Page.emulate', function() { it('should work', async({page, server}) => { await page.goto(server.PREFIX + '/mobile.html'); await page.emulate(iPhone); @@ -99,7 +99,7 @@ module.exports.addTests = function({testRunner, expect, product}) { }); }); - describe('Page.emulateMedia', function() { + describe_fails_ffox('Page.emulateMedia', function() { it('should work', async({page, server}) => { expect(await page.evaluate(() => window.matchMedia('screen').matches)).toBe(true); expect(await page.evaluate(() => window.matchMedia('print').matches)).toBe(false); diff --git a/test/evaluation.spec.js b/test/evaluation.spec.js index bc2dfaea096fb..1df0140930553 100644 --- a/test/evaluation.spec.js +++ b/test/evaluation.spec.js @@ -23,9 +23,9 @@ try { asyncawait = false; } -module.exports.addTests = function({testRunner, expect, FFOX}) { +module.exports.addTests = function({testRunner, expect}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('Page.evaluate', function() { @@ -65,10 +65,10 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { await page.goto(server.PREFIX + '/global-var.html'); expect(await page.evaluate('globalVar')).toBe(123); }); - (FFOX ? xit : it)('should return undefined for objects with symbols', async({page, server}) => { + it_fails_ffox('should return undefined for objects with symbols', async({page, server}) => { expect(await page.evaluate(() => [Symbol('foo4')])).toBe(undefined); }); - (asyncawait ? it : xit)('should work with function shorthands', async({page, server}) => { + (asyncawait ? it_fails_ffox : xit)('should work with function shorthands', async({page, server}) => { // trick node6 transpiler to not touch our object. // TODO(lushnikov): remove eval once Node6 is dropped. const a = eval(`({ @@ -101,7 +101,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { await page.goto(server.EMPTY_PAGE); expect(await frameEvaluation).toBe(42); }); - it('should work from-inside an exposed function', async({page, server}) => { + it_fails_ffox('should work from-inside an exposed function', async({page, server}) => { // Setup inpage callback, which calls Page.evaluate await page.exposeFunction('callController', async function(a, b) { return await page.evaluate((a, b) => a * b, a, b); @@ -158,7 +158,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { it('should properly serialize null fields', async({page}) => { expect(await page.evaluate(() => ({a: undefined}))).toEqual({}); }); - it('should return undefined for non-serializable objects', async({page, server}) => { + it_fails_ffox('should return undefined for non-serializable objects', async({page, server}) => { expect(await page.evaluate(() => window)).toBe(undefined); expect(await page.evaluate(() => [Symbol('foo4')])).toBe(undefined); }); @@ -189,7 +189,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { const text = await page.evaluate(e => e.textContent, element); expect(text).toBe('42'); }); - it('should throw if underlying element was disposed', async({page, server}) => { + it_fails_ffox('should throw if underlying element was disposed', async({page, server}) => { await page.setContent('
39
'); const element = await page.$('section'); expect(element).toBeTruthy(); @@ -198,7 +198,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { await page.evaluate(e => e.textContent, element).catch(e => error = e); expect(error.message).toContain('JSHandle is disposed'); }); - it('should throw if elementHandles are from other frames', async({page, server}) => { + it_fails_ffox('should throw if elementHandles are from other frames', async({page, server}) => { await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); const bodyHandle = await page.frames()[1].$('body'); let error = null; @@ -206,7 +206,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { expect(error).toBeTruthy(); expect(error.message).toContain('JSHandles can be evaluated only in the context they were created'); }); - it('should simulate a user gesture', async({page, server}) => { + it_fails_ffox('should simulate a user gesture', async({page, server}) => { await page.evaluate(playAudio); // also test evaluating strings await page.evaluate(`(${playAudio})()`); @@ -218,7 +218,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { return audio.play(); } }); - it('should throw a nice error after a navigation', async({page, server}) => { + it_fails_ffox('should throw a nice error after a navigation', async({page, server}) => { const executionContext = await page.mainFrame().executionContext(); await Promise.all([ diff --git a/test/frame.spec.js b/test/frame.spec.js index 07bd123fa8c64..2ed0c19c86456 100644 --- a/test/frame.spec.js +++ b/test/frame.spec.js @@ -18,11 +18,11 @@ const utils = require('./utils'); module.exports.addTests = function({testRunner, expect}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('Frame.executionContext', function() { - it('should work', async({page, server}) => { + it_fails_ffox('should work', async({page, server}) => { await page.goto(server.EMPTY_PAGE); await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); expect(page.frames().length).toBe(2); @@ -58,7 +58,7 @@ module.exports.addTests = function({testRunner, expect}) { }); describe('Frame.evaluate', function() { - it('should throw for detached frames', async({page, server}) => { + it_fails_ffox('should throw for detached frames', async({page, server}) => { const frame1 = await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); await utils.detachFrame(page, 'frame1'); let error = null; @@ -68,7 +68,7 @@ module.exports.addTests = function({testRunner, expect}) { }); describe('Frame Management', function() { - it('should handle nested frames', async({page, server}) => { + it_fails_ffox('should handle nested frames', async({page, server}) => { await page.goto(server.PREFIX + '/frames/nested-frames.html'); expect(utils.dumpFrames(page.mainFrame())).toBeGolden('nested-frames.txt'); }); diff --git a/test/ignorehttpserrors.spec.js b/test/ignorehttpserrors.spec.js index 3a39a415e40d8..ed66e28e0e46d 100644 --- a/test/ignorehttpserrors.spec.js +++ b/test/ignorehttpserrors.spec.js @@ -16,7 +16,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, puppeteer}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('ignoreHTTPSErrors', function() { beforeAll(async state => { @@ -34,7 +34,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p await state.page.close(); delete state.page; }); - it('should work', async({page, httpsServer}) => { + it_fails_ffox('should work', async({page, httpsServer}) => { let error = null; const response = await page.goto(httpsServer.EMPTY_PAGE).catch(e => error = e); expect(error).toBe(null); @@ -42,7 +42,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p expect(response.securityDetails()).toBeTruthy(); expect(response.securityDetails().protocol()).toBe('TLS 1.2'); }); - it('Network redirects should report SecurityDetails', async({page, httpsServer}) => { + it_fails_ffox('Network redirects should report SecurityDetails', async({page, httpsServer}) => { httpsServer.setRedirect('/plzredirect', '/empty.html'); const responses = []; page.on('response', response => responses.push(response)); @@ -52,7 +52,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p const securityDetails = responses[0].securityDetails(); expect(securityDetails.protocol()).toBe('TLS 1.2'); }); - it('should work with request interception', async({page, server, httpsServer}) => { + it_fails_ffox('should work with request interception', async({page, server, httpsServer}) => { await page.setRequestInterception(true); page.on('request', request => request.continue()); const response = await page.goto(httpsServer.EMPTY_PAGE); diff --git a/test/input.spec.js b/test/input.spec.js index 34fcc613eac20..a27236a62627b 100644 --- a/test/input.spec.js +++ b/test/input.spec.js @@ -18,10 +18,10 @@ const path = require('path'); module.exports.addTests = function({testRunner, expect}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('input', function() { - it('should upload the file', async({page, server}) => { + it_fails_ffox('should upload the file', async({page, server}) => { await page.goto(server.PREFIX + '/input/fileupload.html'); const filePath = path.relative(process.cwd(), __dirname + '/assets/file-to-upload.txt'); const input = await page.$('input'); @@ -34,7 +34,7 @@ module.exports.addTests = function({testRunner, expect}) { return promise.then(() => reader.result); }, input)).toBe('contents of the file'); }); - it('keyboard.modifiers()', async({page, server}) => { + it_fails_ffox('keyboard.modifiers()', async({page, server}) => { const keyboard = page.keyboard; expect(keyboard._modifiers).toBe(0); await keyboard.down('Shift'); diff --git a/test/jshandle.spec.js b/test/jshandle.spec.js index 15b3b6dfccf06..ed0231741b67b 100644 --- a/test/jshandle.spec.js +++ b/test/jshandle.spec.js @@ -16,11 +16,11 @@ module.exports.addTests = function({testRunner, expect}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('Page.evaluateHandle', function() { - it('should work', async({page, server}) => { + it_fails_ffox('should work', async({page, server}) => { const windowHandle = await page.evaluateHandle(() => window); expect(windowHandle).toBeTruthy(); }); @@ -34,7 +34,7 @@ module.exports.addTests = function({testRunner, expect}) { const isFive = await page.evaluate(e => Object.is(e, 5), aHandle); expect(isFive).toBeTruthy(); }); - it('should warn on nested object handles', async({page, server}) => { + it_fails_ffox('should warn on nested object handles', async({page, server}) => { const aHandle = await page.evaluateHandle(() => document.body); let error = null; await page.evaluateHandle( @@ -81,12 +81,12 @@ module.exports.addTests = function({testRunner, expect}) { const json = await aHandle.jsonValue(); expect(json).toEqual({foo: 'bar'}); }); - it('should not work with dates', async({page, server}) => { + it_fails_ffox('should not work with dates', async({page, server}) => { const dateHandle = await page.evaluateHandle(() => new Date('2017-09-26T00:00:00.000Z')); const json = await dateHandle.jsonValue(); expect(json).toEqual({}); }); - it('should throw for circular objects', async({page, server}) => { + it_fails_ffox('should throw for circular objects', async({page, server}) => { const windowHandle = await page.evaluateHandle('window'); let error = null; await windowHandle.jsonValue().catch(e => error = e); diff --git a/test/keyboard.spec.js b/test/keyboard.spec.js index dad6c9a9f5c10..1b13abf79c632 100644 --- a/test/keyboard.spec.js +++ b/test/keyboard.spec.js @@ -19,7 +19,7 @@ const os = require('os'); module.exports.addTests = function({testRunner, expect, FFOX}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('Keyboard', function() { @@ -55,7 +55,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { await page.keyboard.press('Backspace'); expect(await page.evaluate(() => document.querySelector('textarea').value)).toBe('Hello World!'); }); - it('should send a character with ElementHandle.press', async({page, server}) => { + it_fails_ffox('should send a character with ElementHandle.press', async({page, server}) => { await page.goto(server.PREFIX + '/input/textarea.html'); const textarea = await page.$('textarea'); await textarea.press('a', {text: 'f'}); @@ -75,7 +75,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { await page.keyboard.sendCharacter('a'); expect(await page.evaluate(() => document.querySelector('textarea').value)).toBe('嗨a'); }); - it('should report shiftKey', async({page, server}) => { + it_fails_ffox('should report shiftKey', async({page, server}) => { await page.goto(server.PREFIX + '/input/keyboard.html'); const keyboard = page.keyboard; const codeForKey = {'Shift': 16, 'Alt': 18, 'Meta': 91, 'Control': 17}; @@ -95,7 +95,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { expect(await page.evaluate(() => getResult())).toBe('Keyup: ' + modifierKey + ' ' + modifierKey + 'Left ' + codeForKey[modifierKey] + ' []'); } }); - it('should report multiple modifiers', async({page, server}) => { + it_fails_ffox('should report multiple modifiers', async({page, server}) => { await page.goto(server.PREFIX + '/input/keyboard.html'); const keyboard = page.keyboard; await keyboard.down('Control'); @@ -111,7 +111,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { await keyboard.up('Meta'); expect(await page.evaluate(() => getResult())).toBe('Keyup: Meta MetaLeft 91 []'); }); - it('should send proper codes while typing', async({page, server}) => { + it_fails_ffox('should send proper codes while typing', async({page, server}) => { await page.goto(server.PREFIX + '/input/keyboard.html'); await page.keyboard.type('!'); expect(await page.evaluate(() => getResult())).toBe( @@ -124,7 +124,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { 'Keypress: ^ Digit6 94 94 94 []', 'Keyup: ^ Digit6 54 []'].join('\n')); }); - it('should send proper codes while typing with shift', async({page, server}) => { + it_fails_ffox('should send proper codes while typing with shift', async({page, server}) => { await page.goto(server.PREFIX + '/input/keyboard.html'); const keyboard = page.keyboard; await keyboard.down('Shift'); @@ -136,7 +136,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { 'Keyup: ~ Backquote 192 [Shift]'].join('\n')); await keyboard.up('Shift'); }); - it('should not type canceled events', async({page, server}) => { + it_fails_ffox('should not type canceled events', async({page, server}) => { await page.goto(server.PREFIX + '/input/textarea.html'); await page.focus('textarea'); await page.evaluate(() => { diff --git a/test/launcher.spec.js b/test/launcher.spec.js index 0a1321d5957e8..06da1444fc96e 100644 --- a/test/launcher.spec.js +++ b/test/launcher.spec.js @@ -25,8 +25,8 @@ const TMP_FOLDER = path.join(os.tmpdir(), 'pptr_tmp_folder-'); const utils = require('./utils'); module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, puppeteer}) { - const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {describe, xdescribe, fdescribe, describe_fails_ffox} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('Puppeteer', function() { @@ -59,7 +59,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p await rmAsync(downloadsFolder); }); }); - describe('Browser.disconnect', function() { + describe_fails_ffox('Browser.disconnect', function() { it('should reject navigation when browser closes', async({server}) => { server.setRoute('/one-style.css', () => {}); const browser = await puppeteer.launch(defaultBrowserOptions); @@ -94,13 +94,13 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p await neverResolves; expect(error.message).toContain('Protocol error'); }); - it('should reject if executable path is invalid', async({server}) => { + it_fails_ffox('should reject if executable path is invalid', async({server}) => { let waitError = null; const options = Object.assign({}, defaultBrowserOptions, {executablePath: 'random-invalid-path'}); await puppeteer.launch(options).catch(e => waitError = e); expect(waitError.message.startsWith('Failed to launch chrome! spawn random-invalid-path ENOENT')).toBe(true); }); - it('userDataDir option', async({server}) => { + it_fails_ffox('userDataDir option', async({server}) => { const userDataDir = await mkdtempAsync(TMP_FOLDER); const options = Object.assign({userDataDir}, defaultBrowserOptions); const browser = await puppeteer.launch(options); @@ -112,7 +112,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p // This might throw. See https://github.com/GoogleChrome/puppeteer/issues/2778 await rmAsync(userDataDir).catch(e => {}); }); - it('userDataDir argument', async({server}) => { + it_fails_ffox('userDataDir argument', async({server}) => { const userDataDir = await mkdtempAsync(TMP_FOLDER); const options = Object.assign({}, defaultBrowserOptions); options.args = [`--user-data-dir=${userDataDir}`].concat(options.args || []); @@ -123,7 +123,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p // This might throw. See https://github.com/GoogleChrome/puppeteer/issues/2778 await rmAsync(userDataDir).catch(e => {}); }); - it('userDataDir option should restore state', async({server}) => { + it_fails_ffox('userDataDir option should restore state', async({server}) => { const userDataDir = await mkdtempAsync(TMP_FOLDER); const options = Object.assign({userDataDir}, defaultBrowserOptions); const browser = await puppeteer.launch(options); @@ -140,7 +140,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p // This might throw. See https://github.com/GoogleChrome/puppeteer/issues/2778 await rmAsync(userDataDir).catch(e => {}); }); - it('userDataDir option should restore cookies', async({server}) => { + it_fails_ffox('userDataDir option should restore cookies', async({server}) => { const userDataDir = await mkdtempAsync(TMP_FOLDER); const options = Object.assign({userDataDir}, defaultBrowserOptions); const browser = await puppeteer.launch(options); @@ -157,7 +157,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p // This might throw. See https://github.com/GoogleChrome/puppeteer/issues/2778 await rmAsync(userDataDir).catch(e => {}); }); - it('should return the default chrome arguments', async() => { + it_fails_ffox('should return the default chrome arguments', async() => { expect(puppeteer.defaultArgs()).toContain('--no-first-run'); expect(puppeteer.defaultArgs()).toContain('--headless'); expect(puppeteer.defaultArgs({headless: false})).not.toContain('--headless'); @@ -175,7 +175,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p expect(dumpioData).toContain(dumpioTextToLog); }); - it('should close the browser when the node process closes', async({ server }) => { + it_fails_ffox('should close the browser when the node process closes', async({ server }) => { const {spawn, execSync} = require('child_process'); const res = spawn('node', [path.join(__dirname, 'fixtures', 'closeme.js'), utils.projectRoot(), JSON.stringify(defaultBrowserOptions)]); let wsEndPointCallback; @@ -196,7 +196,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p process.kill(res.pid); await Promise.all(promises); }); - it('should support the pipe option', async() => { + it_fails_ffox('should support the pipe option', async() => { const options = Object.assign({pipe: true}, defaultBrowserOptions); const browser = await puppeteer.launch(options); expect((await browser.pages()).length).toBe(1); @@ -206,7 +206,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p await page.close(); await browser.close(); }); - it('should support the pipe argument', async() => { + it_fails_ffox('should support the pipe argument', async() => { const options = Object.assign({}, defaultBrowserOptions); options.args = ['--remote-debugging-pipe'].concat(options.args || []); const browser = await puppeteer.launch(options); @@ -216,7 +216,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p await page.close(); await browser.close(); }); - it('should fire "disconnected" when closing with pipe', async() => { + it_fails_ffox('should fire "disconnected" when closing with pipe', async() => { const options = Object.assign({pipe: true}, defaultBrowserOptions); const browser = await puppeteer.launch(options); const disconnectedEventPromise = new Promise(resolve => browser.once('disconnected', resolve)); @@ -233,7 +233,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p await page.close(); await browser.close(); }); - it('should filter out ignored default arguments', async() => { + it_fails_ffox('should filter out ignored default arguments', async() => { // Make sure we launch with `--enable-automation` by default. const defaultArgs = puppeteer.defaultArgs(); const browser = await puppeteer.launch(Object.assign({}, defaultBrowserOptions, { @@ -246,13 +246,13 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p expect(spawnargs.indexOf(defaultArgs[2])).toBe(-1); await browser.close(); }); - it('should have default url when launching browser', async function() { + it_fails_ffox('should have default url when launching browser', async function() { const browser = await puppeteer.launch(defaultBrowserOptions); const pages = (await browser.pages()).map(page => page.url()); expect(pages).toEqual(['about:blank']); await browser.close(); }); - it('should have custom url when launching browser', async function({server}) { + it_fails_ffox('should have custom url when launching browser', async function({server}) { const customUrl = server.PREFIX + '/empty.html'; const options = Object.assign({}, defaultBrowserOptions); options.args = [customUrl].concat(options.args || []); @@ -300,7 +300,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p await browser.close(); }); }); - describe('Puppeteer.connect', function() { + describe_fails_ffox('Puppeteer.connect', function() { it('should be able to connect multiple times to the same browser', async({server}) => { const originalBrowser = await puppeteer.launch(defaultBrowserOptions); const browser = await puppeteer.connect({ @@ -408,7 +408,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p }); }); - describe('Browser.Events.disconnected', function() { + describe_fails_ffox('Browser.Events.disconnected', function() { it('should be emitted when: browser gets closed, disconnected or underlying websocket gets closed', async() => { const originalBrowser = await puppeteer.launch(defaultBrowserOptions); const browserWSEndpoint = originalBrowser.wsEndpoint(); diff --git a/test/mouse.spec.js b/test/mouse.spec.js index 5cf6df56c7fdf..859cfc4b38ede 100644 --- a/test/mouse.spec.js +++ b/test/mouse.spec.js @@ -26,7 +26,7 @@ function dimensions() { module.exports.addTests = function({testRunner, expect, FFOX}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('Mouse', function() { @@ -54,7 +54,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { expect(event.isTrusted).toBe(true); expect(event.button).toBe(0); }); - it('should resize the textarea', async({page, server}) => { + it_fails_ffox('should resize the textarea', async({page, server}) => { await page.goto(server.PREFIX + '/input/textarea.html'); const {x, y, width, height} = await page.evaluate(dimensions); const mouse = page.mouse; @@ -66,7 +66,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { expect(newDimensions.width).toBe(width + 104); expect(newDimensions.height).toBe(height + 104); }); - it('should select the text with mouse', async({page, server}) => { + it_fails_ffox('should select the text with mouse', async({page, server}) => { await page.goto(server.PREFIX + '/input/textarea.html'); await page.focus('textarea'); const text = 'This is the text that we are going to try to select. Let\'s see how it goes.'; @@ -88,13 +88,13 @@ module.exports.addTests = function({testRunner, expect, FFOX}) { await page.hover('#button-91'); expect(await page.evaluate(() => document.querySelector('button:hover').id)).toBe('button-91'); }); - it('should trigger hover state with removed window.Node', async({page, server}) => { + it_fails_ffox('should trigger hover state with removed window.Node', async({page, server}) => { await page.goto(server.PREFIX + '/input/scrollable.html'); await page.evaluate(() => delete window.Node); await page.hover('#button-6'); expect(await page.evaluate(() => document.querySelector('button:hover').id)).toBe('button-6'); }); - it('should set modifier keys on click', async({page, server}) => { + it_fails_ffox('should set modifier keys on click', async({page, server}) => { await page.goto(server.PREFIX + '/input/scrollable.html'); await page.evaluate(() => document.querySelector('#button-3').addEventListener('mousedown', e => window.lastEvent = e, true)); const modifiers = {'Shift': 'shiftKey', 'Control': 'ctrlKey', 'Alt': 'altKey', 'Meta': 'metaKey'}; diff --git a/test/navigation.spec.js b/test/navigation.spec.js index 63f8440be5874..f0ee5893d0503 100644 --- a/test/navigation.spec.js +++ b/test/navigation.spec.js @@ -16,9 +16,9 @@ const utils = require('./utils'); -module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { +module.exports.addTests = function({testRunner, expect, Errors}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; const {TimeoutError} = Errors; @@ -33,11 +33,11 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { await page.goto(server.PREFIX + '/redirect/1.html'); expect(page.url()).toBe(server.EMPTY_PAGE); }); - it('should navigate to about:blank', async({page, server}) => { + it_fails_ffox('should navigate to about:blank', async({page, server}) => { const response = await page.goto('about:blank'); expect(response).toBe(null); }); - it('should return response when page changes its URL after load', async({page, server}) => { + it_fails_ffox('should return response when page changes its URL after load', async({page, server}) => { const response = await page.goto(server.PREFIX + '/historyapi.html'); expect(response.status()).toBe(200); }); @@ -48,7 +48,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { }); await page.goto(server.PREFIX + '/frames/one-frame.html'); }); - it('should fail when server returns 204', async({page, server}) => { + it_fails_ffox('should fail when server returns 204', async({page, server}) => { server.setRoute('/empty.html', (req, res) => { res.statusCode = 204; res.end(); @@ -58,12 +58,12 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(error).not.toBe(null); expect(error.message).toContain('net::ERR_ABORTED'); }); - it('should navigate to empty page with domcontentloaded', async({page, server}) => { + it_fails_ffox('should navigate to empty page with domcontentloaded', async({page, server}) => { const response = await page.goto(server.EMPTY_PAGE, {waitUntil: 'domcontentloaded'}); expect(response.status()).toBe(200); expect(response.securityDetails()).toBe(null); }); - it('should work when page calls history API in beforeunload', async({page, server}) => { + it_fails_ffox('should work when page calls history API in beforeunload', async({page, server}) => { await page.goto(server.EMPTY_PAGE); await page.evaluate(() => { window.addEventListener('beforeunload', () => history.replaceState(null, 'initial', window.location.href), false); @@ -71,20 +71,20 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { const response = await page.goto(server.PREFIX + '/grid.html'); expect(response.status()).toBe(200); }); - (FFOX ? xit : it)('should navigate to empty page with networkidle0', async({page, server}) => { + it_fails_ffox('should navigate to empty page with networkidle0', async({page, server}) => { const response = await page.goto(server.EMPTY_PAGE, {waitUntil: 'networkidle0'}); expect(response.status()).toBe(200); }); - (FFOX ? xit : it)('should navigate to empty page with networkidle2', async({page, server}) => { + it_fails_ffox('should navigate to empty page with networkidle2', async({page, server}) => { const response = await page.goto(server.EMPTY_PAGE, {waitUntil: 'networkidle2'}); expect(response.status()).toBe(200); }); - it('should fail when navigating to bad url', async({page, server}) => { + it_fails_ffox('should fail when navigating to bad url', async({page, server}) => { let error = null; await page.goto('asdfasdf').catch(e => error = e); expect(error.message).toContain('Cannot navigate to invalid URL'); }); - it('should fail when navigating to bad SSL', async({page, httpsServer}) => { + it_fails_ffox('should fail when navigating to bad SSL', async({page, httpsServer}) => { // Make sure that network events do not emit 'undefined'. // @see https://crbug.com/750469 page.on('request', request => expect(request).toBeTruthy()); @@ -94,19 +94,19 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { await page.goto(httpsServer.EMPTY_PAGE).catch(e => error = e); expect(error.message).toContain('net::ERR_CERT_AUTHORITY_INVALID'); }); - it('should fail when navigating to bad SSL after redirects', async({page, server, httpsServer}) => { + it_fails_ffox('should fail when navigating to bad SSL after redirects', async({page, server, httpsServer}) => { server.setRedirect('/redirect/1.html', '/redirect/2.html'); server.setRedirect('/redirect/2.html', '/empty.html'); let error = null; await page.goto(httpsServer.PREFIX + '/redirect/1.html').catch(e => error = e); expect(error.message).toContain('net::ERR_CERT_AUTHORITY_INVALID'); }); - it('should throw if networkidle is passed as an option', async({page, server}) => { + it_fails_ffox('should throw if networkidle is passed as an option', async({page, server}) => { let error = null; await page.goto(server.EMPTY_PAGE, {waitUntil: 'networkidle'}).catch(err => error = err); expect(error.message).toContain('"networkidle" option is no longer supported'); }); - it('should fail when main resources failed to load', async({page, server}) => { + it_fails_ffox('should fail when main resources failed to load', async({page, server}) => { let error = null; await page.goto('http://localhost:44123/non-existing-url').catch(e => error = e); expect(error.message).toContain('net::ERR_CONNECTION_REFUSED'); @@ -119,7 +119,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(error.message).toContain('Navigation Timeout Exceeded: 1ms'); expect(error).toBeInstanceOf(TimeoutError); }); - it('should fail when exceeding default maximum navigation timeout', async({page, server}) => { + it_fails_ffox('should fail when exceeding default maximum navigation timeout', async({page, server}) => { // Hang for request to the empty.html server.setRoute('/empty.html', (req, res) => { }); let error = null; @@ -128,7 +128,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(error.message).toContain('Navigation Timeout Exceeded: 1ms'); expect(error).toBeInstanceOf(TimeoutError); }); - it('should fail when exceeding default maximum timeout', async({page, server}) => { + it_fails_ffox('should fail when exceeding default maximum timeout', async({page, server}) => { // Hang for request to the empty.html server.setRoute('/empty.html', (req, res) => { }); let error = null; @@ -137,7 +137,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(error.message).toContain('Navigation Timeout Exceeded: 1ms'); expect(error).toBeInstanceOf(TimeoutError); }); - it('should prioritize default navigation timeout over default timeout', async({page, server}) => { + it_fails_ffox('should prioritize default navigation timeout over default timeout', async({page, server}) => { // Hang for request to the empty.html server.setRoute('/empty.html', (req, res) => { }); let error = null; @@ -155,20 +155,20 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(error).toBe(null); expect(loaded).toBe(true); }); - it('should work when navigating to valid url', async({page, server}) => { + it_fails_ffox('should work when navigating to valid url', async({page, server}) => { const response = await page.goto(server.EMPTY_PAGE); expect(response.ok()).toBe(true); }); - it('should work when navigating to data url', async({page, server}) => { + it_fails_ffox('should work when navigating to data url', async({page, server}) => { const response = await page.goto('data:text/html,hello'); expect(response.ok()).toBe(true); }); - it('should work when navigating to 404', async({page, server}) => { + it_fails_ffox('should work when navigating to 404', async({page, server}) => { const response = await page.goto(server.PREFIX + '/not-found'); expect(response.ok()).toBe(false); expect(response.status()).toBe(404); }); - it('should return last response in redirect chain', async({page, server}) => { + it_fails_ffox('should return last response in redirect chain', async({page, server}) => { server.setRedirect('/redirect/1.html', '/redirect/2.html'); server.setRedirect('/redirect/2.html', '/redirect/3.html'); server.setRedirect('/redirect/3.html', server.EMPTY_PAGE); @@ -176,7 +176,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(response.ok()).toBe(true); expect(response.url()).toBe(server.EMPTY_PAGE); }); - (FFOX ? xit : it)('should wait for network idle to succeed navigation', async({page, server}) => { + it_fails_ffox('should wait for network idle to succeed navigation', async({page, server}) => { let responses = []; // Hold on to a bunch of requests without answering. server.setRoute('/fetch-request-a.js', (req, res) => responses.push(res)); @@ -242,7 +242,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { process.removeListener('warning', warningHandler); expect(warning).toBe(null); }); - it('should not leak listeners during bad navigation', async({page, server}) => { + it_fails_ffox('should not leak listeners during bad navigation', async({page, server}) => { let warning = null; const warningHandler = w => warning = w; process.on('warning', warningHandler); @@ -251,7 +251,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { process.removeListener('warning', warningHandler); expect(warning).toBe(null); }); - it('should not leak listeners during navigation of 11 pages', async({page, context, server}) => { + it_fails_ffox('should not leak listeners during navigation of 11 pages', async({page, context, server}) => { let warning = null; const warningHandler = w => warning = w; process.on('warning', warningHandler); @@ -263,7 +263,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { process.removeListener('warning', warningHandler); expect(warning).toBe(null); }); - it('should navigate to dataURL and fire dataURL requests', async({page, server}) => { + it_fails_ffox('should navigate to dataURL and fire dataURL requests', async({page, server}) => { const requests = []; page.on('request', request => requests.push(request)); const dataURL = 'data:text/html,
yo
'; @@ -272,7 +272,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(requests.length).toBe(1); expect(requests[0].url()).toBe(dataURL); }); - it('should navigate to URL with hash and fire requests without hash', async({page, server}) => { + it_fails_ffox('should navigate to URL with hash and fire requests without hash', async({page, server}) => { const requests = []; page.on('request', request => requests.push(request)); const response = await page.goto(server.EMPTY_PAGE + '#hash'); @@ -281,7 +281,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(requests.length).toBe(1); expect(requests[0].url()).toBe(server.EMPTY_PAGE); }); - it('should work with self requesting page', async({page, server}) => { + it_fails_ffox('should work with self requesting page', async({page, server}) => { const response = await page.goto(server.PREFIX + '/self-request.html'); expect(response.status()).toBe(200); expect(response.url()).toContain('self-request.html'); @@ -296,7 +296,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { } expect(error.message).toContain(url); }); - it('should send referer', async({page, server}) => { + it_fails_ffox('should send referer', async({page, server}) => { const [request1, request2] = await Promise.all([ server.waitForRequest('/grid.html'), server.waitForRequest('/digits/1.png'), @@ -311,7 +311,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { }); describe('Page.waitForNavigation', function() { - it('should work', async({page, server}) => { + it_fails_ffox('should work', async({page, server}) => { await page.goto(server.EMPTY_PAGE); const [response] = await Promise.all([ page.waitForNavigation(), @@ -340,7 +340,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { await bothFiredPromise; await navigationPromise; }); - it('should work with clicking on anchor links', async({page, server}) => { + it_fails_ffox('should work with clicking on anchor links', async({page, server}) => { await page.goto(server.EMPTY_PAGE); await page.setContent(`foobar`); const [response] = await Promise.all([ @@ -350,7 +350,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(response).toBe(null); expect(page.url()).toBe(server.EMPTY_PAGE + '#foobar'); }); - it('should work with history.pushState()', async({page, server}) => { + it_fails_ffox('should work with history.pushState()', async({page, server}) => { await page.goto(server.EMPTY_PAGE); await page.setContent(` SPA @@ -365,7 +365,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(response).toBe(null); expect(page.url()).toBe(server.PREFIX + '/wow.html'); }); - it('should work with history.replaceState()', async({page, server}) => { + it_fails_ffox('should work with history.replaceState()', async({page, server}) => { await page.goto(server.EMPTY_PAGE); await page.setContent(` SPA @@ -380,7 +380,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(response).toBe(null); expect(page.url()).toBe(server.PREFIX + '/replaced.html'); }); - it('should work with DOM history.back()/history.forward()', async({page, server}) => { + it_fails_ffox('should work with DOM history.back()/history.forward()', async({page, server}) => { await page.goto(server.EMPTY_PAGE); await page.setContent(` back @@ -424,7 +424,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { }); describe('Page.goBack', function() { - it('should work', async({page, server}) => { + it_fails_ffox('should work', async({page, server}) => { await page.goto(server.EMPTY_PAGE); await page.goto(server.PREFIX + '/grid.html'); @@ -457,7 +457,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { }); describe('Frame.goto', function() { - it('should navigate subframes', async({page, server}) => { + it_fails_ffox('should navigate subframes', async({page, server}) => { await page.goto(server.PREFIX + '/frames/one-frame.html'); expect(page.frames()[0].url()).toContain('/frames/one-frame.html'); expect(page.frames()[1].url()).toContain('/frames/frame.html'); @@ -466,7 +466,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(response.ok()).toBe(true); expect(response.frame()).toBe(page.frames()[1]); }); - it('should reject when frame detaches', async({page, server}) => { + it_fails_ffox('should reject when frame detaches', async({page, server}) => { await page.goto(server.PREFIX + '/frames/one-frame.html'); server.setRoute('/empty.html', () => {}); @@ -477,7 +477,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { const error = await navigationPromise; expect(error.message).toBe('Navigating frame was detached'); }); - it('should return matching responses', async({page, server}) => { + it_fails_ffox('should return matching responses', async({page, server}) => { // Disable cache: otherwise, chromium will cache similar requests. await page.setCacheEnabled(false); await page.goto(server.EMPTY_PAGE); @@ -507,7 +507,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { }); describe('Frame.waitForNavigation', function() { - it('should work', async({page, server}) => { + it_fails_ffox('should work', async({page, server}) => { await page.goto(server.PREFIX + '/frames/one-frame.html'); const frame = page.frames()[1]; const [response] = await Promise.all([ @@ -519,7 +519,7 @@ module.exports.addTests = function({testRunner, expect, Errors, FFOX}) { expect(response.frame()).toBe(frame); expect(page.url()).toContain('/frames/one-frame.html'); }); - it('should reject when frame detaches', async({page, server}) => { + it_fails_ffox('should reject when frame detaches', async({page, server}) => { await page.goto(server.PREFIX + '/frames/one-frame.html'); const frame = page.frames()[1]; diff --git a/test/network.spec.js b/test/network.spec.js index 693d05116e85b..e3233f3015d40 100644 --- a/test/network.spec.js +++ b/test/network.spec.js @@ -19,11 +19,11 @@ const path = require('path'); const utils = require('./utils'); module.exports.addTests = function({testRunner, expect}) { - const {describe, xdescribe, fdescribe} = testRunner; + const {describe, xdescribe, fdescribe, describe_fails_ffox} = testRunner; const {it, fit, xit} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; - describe('Network Events', function() { + describe_fails_ffox('Network Events', function() { it('Page.Events.Request', async({page, server}) => { const requests = []; page.on('request', request => requests.push(request)); @@ -213,7 +213,7 @@ module.exports.addTests = function({testRunner, expect}) { }); }); - describe('Request.isNavigationRequest', () => { + describe_fails_ffox('Request.isNavigationRequest', () => { it('should work', async({page, server}) => { const requests = new Map(); page.on('request', request => requests.set(request.url().split('/').pop(), request)); @@ -248,7 +248,7 @@ module.exports.addTests = function({testRunner, expect}) { }); }); - describe('Page.setRequestInterception', function() { + describe_fails_ffox('Page.setRequestInterception', function() { it('should intercept', async({page, server}) => { await page.setRequestInterception(true); page.on('request', request => { @@ -595,7 +595,7 @@ module.exports.addTests = function({testRunner, expect}) { }); }); - describe('Request.respond', function() { + describe_fails_ffox('Request.respond', function() { it('should work', async({page, server}) => { await page.setRequestInterception(true); page.on('request', request => { @@ -632,7 +632,7 @@ module.exports.addTests = function({testRunner, expect}) { }); }); - describe('Page.Events.Request', function() { + describe_fails_ffox('Page.Events.Request', function() { it('should fire', async({page, server}) => { const requests = []; page.on('request', request => requests.push(request)); @@ -648,7 +648,7 @@ module.exports.addTests = function({testRunner, expect}) { }); }); - describe('Page.setExtraHTTPHeaders', function() { + describe_fails_ffox('Page.setExtraHTTPHeaders', function() { it('should work', async({page, server}) => { await page.setExtraHTTPHeaders({ foo: 'bar' @@ -670,7 +670,7 @@ module.exports.addTests = function({testRunner, expect}) { }); }); - describe('Page.authenticate', function() { + describe_fails_ffox('Page.authenticate', function() { it('should work', async({page, server}) => { server.setAuth('/empty.html', 'user', 'pass'); let response = await page.goto(server.EMPTY_PAGE); diff --git a/test/page.spec.js b/test/page.spec.js index dc93e1c9dcb83..82c1a4a47fcfd 100644 --- a/test/page.spec.js +++ b/test/page.spec.js @@ -26,8 +26,8 @@ try { } module.exports.addTests = function({testRunner, expect, headless, Errors, DeviceDescriptors}) { - const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {describe, xdescribe, fdescribe, describe_fails_ffox} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; const {TimeoutError} = Errors; @@ -48,7 +48,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device await newPage.close(); expect(await browser.pages()).not.toContain(newPage); }); - it('should run beforeunload if asked for', async({context, server}) => { + it_fails_ffox('should run beforeunload if asked for', async({context, server}) => { const newPage = await context.newPage(); await newPage.goto(server.PREFIX + '/beforeunload.html'); // We have to interact with a page so that 'beforeunload' handlers @@ -79,7 +79,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - (asyncawait ? describe : xdescribe)('Async stacks', () => { + (asyncawait ? describe_fails_ffox : xdescribe)('Async stacks', () => { it('should work', async({page, server}) => { server.setRoute('/empty.html', (req, res) => { res.statusCode = 204; @@ -92,7 +92,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Page.Events.error', function() { + describe_fails_ffox('Page.Events.error', function() { it('should throw when page crashes', async({page}) => { let error = null; page.on('error', err => error = err); @@ -102,7 +102,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Page.Events.Popup', function() { + describe_fails_ffox('Page.Events.Popup', function() { it('should work', async({page}) => { const [popup] = await Promise.all([ new Promise(x => page.once('popup', x)), @@ -151,7 +151,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('BrowserContext.overridePermissions', function() { + describe_fails_ffox('BrowserContext.overridePermissions', function() { function getPermission(page, name) { return page.evaluate(name => navigator.permissions.query({name}).then(result => result.state), name); } @@ -204,7 +204,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Page.setGeolocation', function() { + describe_fails_ffox('Page.setGeolocation', function() { it('should work', async({page, server, context}) => { await context.overridePermissions(server.PREFIX, ['geolocation']); await page.goto(server.EMPTY_PAGE); @@ -228,7 +228,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Page.setOfflineMode', function() { + describe_fails_ffox('Page.setOfflineMode', function() { it('should work', async({page, server}) => { await page.setOfflineMode(true); let error = null; @@ -247,7 +247,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('ExecutionContext.queryObjects', function() { + describe_fails_ffox('ExecutionContext.queryObjects', function() { it('should work', async({page, server}) => { // Instantiate an object await page.evaluate(() => window.set = new Set(['hello', 'world'])); @@ -322,7 +322,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device ]); expect(message.text()).toBe('JSHandle@object'); }); - it('should trigger correct Log', async({page, server}) => { + it_fails_ffox('should trigger correct Log', async({page, server}) => { await page.goto('about:blank'); const [message] = await Promise.all([ waitEvent(page, 'console'), @@ -331,7 +331,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device expect(message.text()).toContain('No \'Access-Control-Allow-Origin\''); expect(message.type()).toEqual('error'); }); - it('should have location when fetch fails', async({page, server}) => { + it_fails_ffox('should have location when fetch fails', async({page, server}) => { await page.goto(server.EMPTY_PAGE); const [message] = await Promise.all([ waitEvent(page, 'console'), @@ -344,7 +344,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device lineNumber: undefined }); }); - it('should have location for console API calls', async({page, server}) => { + it_fails_ffox('should have location for console API calls', async({page, server}) => { await page.goto(server.EMPTY_PAGE); const [message] = await Promise.all([ waitEvent(page, 'console'), @@ -359,7 +359,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); // @see https://github.com/GoogleChrome/puppeteer/issues/3865 - it('should not throw when there are console messages in detached iframes', async({browser, page, server}) => { + it_fails_ffox('should not throw when there are console messages in detached iframes', async({browser, page, server}) => { await page.goto(server.EMPTY_PAGE); await page.evaluate(async() => { // 1. Create a popup that Puppeteer is not connected to. @@ -385,7 +385,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Page.metrics', function() { + describe_fails_ffox('Page.metrics', function() { it('should get metrics from a page', async({page, server}) => { await page.goto('about:blank'); const metrics = await page.metrics(); @@ -423,7 +423,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device } }); - describe('Page.waitForRequest', function() { + describe_fails_ffox('Page.waitForRequest', function() { it('should work', async({page, server}) => { await page.goto(server.EMPTY_PAGE); const [request] = await Promise.all([ @@ -473,7 +473,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Page.waitForResponse', function() { + describe_fails_ffox('Page.waitForResponse', function() { it('should work', async({page, server}) => { await page.goto(server.EMPTY_PAGE); const [response] = await Promise.all([ @@ -523,7 +523,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Page.exposeFunction', function() { + describe_fails_ffox('Page.exposeFunction', function() { it('should work', async({page, server}) => { await page.exposeFunction('compute', function(a, b) { return a * b; @@ -628,7 +628,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Page.setUserAgent', function() { + describe_fails_ffox('Page.setUserAgent', function() { it('should work', async({page, server}) => { expect(await page.evaluate(() => navigator.userAgent)).toContain('Mozilla'); page.setUserAgent('foobar'); @@ -666,7 +666,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device const result = await page.content(); expect(result).toBe(`${doctype}${expectedOutput}`); }); - it('should respect timeout', async({page, server}) => { + it_fails_ffox('should respect timeout', async({page, server}) => { const imgPath = '/img.png'; // stall for image server.setRoute(imgPath, (req, res) => {}); @@ -674,7 +674,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device await page.setContent(``, {timeout: 1}).catch(e => error = e); expect(error).toBeInstanceOf(TimeoutError); }); - it('should respect default navigation timeout', async({page, server}) => { + it_fails_ffox('should respect default navigation timeout', async({page, server}) => { page.setDefaultNavigationTimeout(1); const imgPath = '/img.png'; // stall for image @@ -683,7 +683,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device await page.setContent(``).catch(e => error = e); expect(error).toBeInstanceOf(TimeoutError); }); - it('should await resources to load', async({page, server}) => { + it_fails_ffox('should await resources to load', async({page, server}) => { const imgPath = '/img.png'; let imgResponse = null; server.setRoute(imgPath, (req, res) => imgResponse = res); @@ -700,7 +700,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Page.setBypassCSP', function() { + describe_fails_ffox('Page.setBypassCSP', function() { it('should bypass CSP meta tag', async({page, server}) => { // Make sure CSP prohibits addScriptTag. await page.goto(server.PREFIX + '/csp.html'); @@ -810,7 +810,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device expect(await page.evaluate(() => __injected)).toBe(35); }); - it('should throw when added with content to the CSP page', async({page, server}) => { + it_fails_ffox('should throw when added with content to the CSP page', async({page, server}) => { await page.goto(server.PREFIX + '/csp.html'); let error = null; await page.addScriptTag({ content: 'window.__injected = 35;' }).catch(e => error = e); @@ -876,7 +876,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device expect(await page.evaluate(`window.getComputedStyle(document.querySelector('body')).getPropertyValue('background-color')`)).toBe('rgb(0, 128, 0)'); }); - it('should throw when added with content to the CSP page', async({page, server}) => { + it_fails_ffox('should throw when added with content to the CSP page', async({page, server}) => { await page.goto(server.PREFIX + '/csp.html'); let error = null; await page.addStyleTag({ content: 'body { background-color: green; }' }).catch(e => error = e); @@ -899,7 +899,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Page.setJavaScriptEnabled', function() { + describe_fails_ffox('Page.setJavaScriptEnabled', function() { it('should work', async({page, server}) => { await page.setJavaScriptEnabled(false); await page.goto('data:text/html, '); @@ -913,7 +913,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Page.setCacheEnabled', function() { + describe_fails_ffox('Page.setCacheEnabled', function() { it('should enable or disable the cache based on the state passed', async({page, server}) => { const responses = new Map(); page.on('response', r => responses.set(r.url().split('/').pop(), r)); @@ -929,7 +929,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); // Printing to pdf is currently only supported in headless - (headless ? describe : xdescribe)('Page.pdf', function() { + (headless ? describe_fails_ffox : xdescribe)('Page.pdf', function() { it('should be able to save file', async({page, server}) => { const outputFile = __dirname + '/assets/output.pdf'; await page.pdf({path: outputFile}); @@ -1022,7 +1022,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device expect(error.message).toContain('Values must be strings'); }); // @see https://github.com/GoogleChrome/puppeteer/issues/3327 - it('should work when re-defining top-level Event class', async({page, server}) => { + it_fails_ffox('should work when re-defining top-level Event class', async({page, server}) => { await page.goto(server.PREFIX + '/input/select.html'); await page.evaluate(() => window.Event = null); await page.select('select', 'blue'); @@ -1031,7 +1031,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Connection', function() { + describe_fails_ffox('Connection', function() { it('should throw nice errors', async function({page}) { const error = await theSourceOfTheProblems().catch(error => error); expect(error.stack).toContain('theSourceOfTheProblems'); @@ -1065,7 +1065,7 @@ module.exports.addTests = function({testRunner, expect, headless, Errors, Device }); }); - describe('Page.browserContext', function() { + describe_fails_ffox('Page.browserContext', function() { it('should return the correct browser instance', async function({page, context, browser}) { expect(page.browserContext()).toBe(context); }); diff --git a/test/puppeteer.spec.js b/test/puppeteer.spec.js index 5b58378de1061..4d3d83d61ca3d 100644 --- a/test/puppeteer.spec.js +++ b/test/puppeteer.spec.js @@ -23,8 +23,8 @@ const YELLOW_COLOR = '\x1b[33m'; const RESET_COLOR = '\x1b[0m'; module.exports.addTests = ({testRunner, product, puppeteer, Errors, DeviceDescriptors}) => { - const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {describe, xdescribe, fdescribe, describe_fails_ffox} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; const CHROME = product === 'Chromium'; @@ -117,8 +117,10 @@ module.exports.addTests = ({testRunner, product, puppeteer, Errors, DeviceDescri // Page-level tests that are given a browser, a context and a page. // Each test is launched in a new browser context. + require('./accessibility.spec.js').addTests(testOptions); require('./browser.spec.js').addTests(testOptions); require('./click.spec.js').addTests(testOptions); + require('./cookies.spec.js').addTests(testOptions); require('./dialog.spec.js').addTests(testOptions); require('./elementhandle.spec.js').addTests(testOptions); require('./emulation.spec.js').addTests(testOptions); @@ -129,19 +131,17 @@ module.exports.addTests = ({testRunner, product, puppeteer, Errors, DeviceDescri require('./keyboard.spec.js').addTests(testOptions); require('./mouse.spec.js').addTests(testOptions); require('./navigation.spec.js').addTests(testOptions); + require('./network.spec.js').addTests(testOptions); require('./page.spec.js').addTests(testOptions); require('./screenshot.spec.js').addTests(testOptions); require('./queryselector.spec.js').addTests(testOptions); require('./target.spec.js').addTests(testOptions); require('./touchscreen.spec.js').addTests(testOptions); require('./waittask.spec.js').addTests(testOptions); + require('./worker.spec.js').addTests(testOptions); if (CHROME) { require('./CDPSession.spec.js').addTests(testOptions); - require('./accessibility.spec.js').addTests(testOptions); - require('./cookies.spec.js').addTests(testOptions); require('./coverage.spec.js').addTests(testOptions); - require('./network.spec.js').addTests(testOptions); - require('./worker.spec.js').addTests(testOptions); require('./chromiumonly.spec.js').addTests(testOptions); } else { require('./firefoxonly.spec.js').addTests(testOptions); diff --git a/test/screenshot.spec.js b/test/screenshot.spec.js index 8fd6ac11658db..b5bbb3f516cc7 100644 --- a/test/screenshot.spec.js +++ b/test/screenshot.spec.js @@ -15,11 +15,11 @@ */ module.exports.addTests = function({testRunner, expect, product}) { - const {describe, xdescribe, fdescribe} = testRunner; + const {describe, xdescribe, fdescribe, describe_fails_ffox} = testRunner; const {it, fit, xit} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; - describe('Page.screenshot', function() { + describe_fails_ffox('Page.screenshot', function() { it('should work', async({page, server}) => { await page.setViewport({width: 500, height: 500}); await page.goto(server.PREFIX + '/grid.html'); @@ -125,7 +125,7 @@ module.exports.addTests = function({testRunner, expect, product}) { }); }); - describe('ElementHandle.screenshot', function() { + describe_fails_ffox('ElementHandle.screenshot', function() { it('should work', async({page, server}) => { await page.setViewport({width: 500, height: 500}); await page.goto(server.PREFIX + '/grid.html'); diff --git a/test/target.spec.js b/test/target.spec.js index 5951f909b1206..8338f50248ccf 100644 --- a/test/target.spec.js +++ b/test/target.spec.js @@ -19,12 +19,12 @@ const {waitEvent} = utils; module.exports.addTests = function({testRunner, expect, puppeteer, Errors}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; const {TimeoutError} = Errors; describe('Target', function() { - it('Browser.targets should return all of the targets', async({page, server, browser}) => { + it_fails_ffox('Browser.targets should return all of the targets', async({page, server, browser}) => { // The pages will be the testing page and the original newtab page const targets = browser.targets(); expect(targets.some(target => target.type() === 'page' && @@ -38,7 +38,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer, Errors}) { expect(allPages).toContain(page); expect(allPages[0]).not.toBe(allPages[1]); }); - it('should contain browser target', async({browser}) => { + it_fails_ffox('should contain browser target', async({browser}) => { const targets = browser.targets(); const browserTarget = targets.find(target => target.type() === 'browser'); expect(browserTarget).toBeTruthy(); @@ -50,7 +50,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer, Errors}) { expect(await originalPage.evaluate(() => ['Hello', 'world'].join(' '))).toBe('Hello world'); expect(await originalPage.$('body')).toBeTruthy(); }); - it('should report when a new page is created and closed', async({page, server, context}) => { + it_fails_ffox('should report when a new page is created and closed', async({page, server, context}) => { const otherPagePromise = new Promise(fulfill => context.once('targetcreated', target => fulfill(target.page()))); await page.evaluate(url => window.open(url), server.CROSS_PROCESS_PREFIX); const otherPage = await otherPagePromise; @@ -71,7 +71,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer, Errors}) { expect(allPages).toContain(page); expect(allPages).not.toContain(otherPage); }); - it('should report when a service worker is created and destroyed', async({page, server, context}) => { + it_fails_ffox('should report when a service worker is created and destroyed', async({page, server, context}) => { await page.goto(server.EMPTY_PAGE); const createdTarget = new Promise(fulfill => context.once('targetcreated', target => fulfill(target))); @@ -94,7 +94,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer, Errors}) { await page.goto(server.EMPTY_PAGE); expect((await changedTarget).url()).toBe(server.EMPTY_PAGE); }); - it('should not report uninitialized pages', async({page, server, context}) => { + it_fails_ffox('should not report uninitialized pages', async({page, server, context}) => { let targetChanged = false; const listener = () => targetChanged = true; context.on('targetchanged', listener); @@ -132,7 +132,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer, Errors}) { // Cleanup. await newPage.close(); }); - it('should have an opener', async({page, server, context}) => { + it_fails_ffox('should have an opener', async({page, server, context}) => { await page.goto(server.EMPTY_PAGE); const [createdTarget] = await Promise.all([ new Promise(fulfill => context.once('targetcreated', target => fulfill(target))), diff --git a/test/test.js b/test/test.js index 8f0174e390362..07e02bb7399d2 100644 --- a/test/test.js +++ b/test/test.js @@ -79,6 +79,8 @@ const CHROMIUM_NO_COVERAGE = new Set([ ]); if (process.env.BROWSER !== 'firefox') { + testRunner.addTestDSL('it_fails_ffox', 'run'); + testRunner.addSuiteDSL('describe_fails_ffox', 'run'); describe('Chromium', () => { require('./puppeteer.spec.js').addTests({ product: 'Chromium', @@ -91,6 +93,9 @@ if (process.env.BROWSER !== 'firefox') { utils.recordAPICoverage(testRunner, require('../lib/api'), CHROMIUM_NO_COVERAGE); }); } else { + const FFOX_SKIPPED_TESTS = Symbol('FFOX_SKIPPED_TESTS'); + testRunner.addTestDSL('it_fails_ffox', 'skip', FFOX_SKIPPED_TESTS); + testRunner.addSuiteDSL('describe_fails_ffox', 'skip', FFOX_SKIPPED_TESTS); describe('Firefox', () => { require('./puppeteer.spec.js').addTests({ product: 'Firefox', @@ -100,6 +105,24 @@ if (process.env.BROWSER !== 'firefox') { testRunner, }); }); + + if (process.argv.indexOf('--firefox-status') !== -1) { + const allTests = testRunner.tests(); + const ffoxTests = allTests.filter(test => { + if (test.comment === FFOX_SKIPPED_TESTS) + return false; + for (let suite = test.suite; suite; suite = suite.parentSuite) { + if (suite.comment === FFOX_SKIPPED_TESTS) + return false; + } + return true; + }); + console.log(JSON.stringify({ + allTests: allTests.length, + firefoxTests: ffoxTests.length + })); + process.exit(0); + } } if (process.env.CI && testRunner.hasFocusedTestsOrSuites()) { diff --git a/test/touchscreen.spec.js b/test/touchscreen.spec.js index 0debb5e65640f..11f537f49d1cf 100644 --- a/test/touchscreen.spec.js +++ b/test/touchscreen.spec.js @@ -16,10 +16,10 @@ module.exports.addTests = function({testRunner, expect}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; describe('Touchscreen', function() { - it('should tap the button', async({page, server}) => { + it_fails_ffox('should tap the button', async({page, server}) => { await page.goto(server.PREFIX + '/input/button.html'); await page.tap('button'); expect(await page.evaluate(() => result)).toBe('Clicked'); diff --git a/test/waittask.spec.js b/test/waittask.spec.js index 7ea29b5eb1511..94c14982a1b7e 100644 --- a/test/waittask.spec.js +++ b/test/waittask.spec.js @@ -25,7 +25,7 @@ try { module.exports.addTests = function({testRunner, expect, product, Errors}) { const {describe, xdescribe, fdescribe} = testRunner; - const {it, fit, xit} = testRunner; + const {it, fit, xit, it_fails_ffox} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; const {TimeoutError} = Errors; @@ -60,7 +60,7 @@ module.exports.addTests = function({testRunner, expect, product, Errors}) { await page.waitFor(timeout); expect(Date.now() - startTime).not.toBeLessThan(timeout / 2); }); - it('should work with multiline body', async({page, server}) => { + it_fails_ffox('should work with multiline body', async({page, server}) => { const result = await page.waitForFunction(` (() => true)() `); @@ -121,7 +121,7 @@ module.exports.addTests = function({testRunner, expect, product, Errors}) { await page.evaluate(() => window.__FOO = 'hit'); await watchdog; }); - it('should work with strict CSP policy', async({page, server}) => { + it_fails_ffox('should work with strict CSP policy', async({page, server}) => { server.setCSP('/empty.html', 'script-src ' + server.PREFIX); await page.goto(server.EMPTY_PAGE); let error = null; @@ -173,7 +173,7 @@ module.exports.addTests = function({testRunner, expect, product, Errors}) { expect(error.message).toContain('waiting for function failed: timeout'); expect(error).toBeInstanceOf(TimeoutError); }); - it('should respect default timeout', async({page}) => { + it_fails_ffox('should respect default timeout', async({page}) => { page.setDefaultTimeout(1); let error = null; await page.waitForFunction('false').catch(e => error = e); @@ -223,7 +223,7 @@ module.exports.addTests = function({testRunner, expect, product, Errors}) { await frame.waitForSelector('div'); }); - it('should work with removed MutationObserver', async({page, server}) => { + it_fails_ffox('should work with removed MutationObserver', async({page, server}) => { await page.evaluate(() => delete window.MutationObserver); const [handle] = await Promise.all([ page.waitForSelector('.zombo'), @@ -251,7 +251,7 @@ module.exports.addTests = function({testRunner, expect, product, Errors}) { await watchdog; }); - it('Page.waitForSelector is shortcut for main frame', async({page, server}) => { + it_fails_ffox('Page.waitForSelector is shortcut for main frame', async({page, server}) => { await page.goto(server.EMPTY_PAGE); await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE2); const otherFrame = page.frames()[1]; @@ -262,7 +262,7 @@ module.exports.addTests = function({testRunner, expect, product, Errors}) { expect(eHandle.executionContext().frame()).toBe(page.mainFrame()); }); - it('should run in specified frame', async({page, server}) => { + it_fails_ffox('should run in specified frame', async({page, server}) => { await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE2); await utils.attachFrame(page, 'frame2', server.EMPTY_PAGE2); const frame1 = page.frames()[1]; @@ -347,7 +347,7 @@ module.exports.addTests = function({testRunner, expect, product, Errors}) { expect(await waitForSelector).toBe(true); expect(divRemoved).toBe(true); }); - it('should return null if waiting to hide non-existing element', async({page, server}) => { + it_fails_ffox('should return null if waiting to hide non-existing element', async({page, server}) => { const handle = await page.waitForSelector('non-existing', { hidden: true }); expect(handle).toBe(null); }); @@ -401,7 +401,7 @@ module.exports.addTests = function({testRunner, expect, product, Errors}) { expect(error.message).toContain('waiting for XPath "//div" failed: timeout'); expect(error).toBeInstanceOf(TimeoutError); }); - it('should run in specified frame', async({page, server}) => { + it_fails_ffox('should run in specified frame', async({page, server}) => { await utils.attachFrame(page, 'frame1', server.EMPTY_PAGE); await utils.attachFrame(page, 'frame2', server.EMPTY_PAGE); const frame1 = page.frames()[1]; diff --git a/test/worker.spec.js b/test/worker.spec.js index a4ccb62fb5203..790494ba4bc5f 100644 --- a/test/worker.spec.js +++ b/test/worker.spec.js @@ -2,11 +2,11 @@ const utils = require('./utils'); const {waitEvent} = utils; module.exports.addTests = function({testRunner, expect}) { - const {describe, xdescribe, fdescribe} = testRunner; + const {describe, xdescribe, fdescribe, describe_fails_ffox} = testRunner; const {it, fit, xit} = testRunner; const {beforeAll, beforeEach, afterAll, afterEach} = testRunner; - describe('Workers', function() { + describe_fails_ffox('Workers', function() { it('Page.workers', async function({page, server}) { await Promise.all([ new Promise(x => page.once('workercreated', x)), diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js index c752702c79c57..ef6093dd89dc8 100644 --- a/utils/testrunner/TestRunner.js +++ b/utils/testrunner/TestRunner.js @@ -97,13 +97,14 @@ const TestResult = { }; class Test { - constructor(suite, name, callback, declaredMode, timeout) { + constructor(suite, name, callback, declaredMode, timeout, comment) { this.suite = suite; this.name = name; this.fullName = (suite.fullName + ' ' + name).trim(); this.declaredMode = declaredMode; this._userCallback = new UserCallback(callback, timeout); this.location = this._userCallback.location; + this.comment = comment; // Test results this.result = null; @@ -114,11 +115,12 @@ class Test { } class Suite { - constructor(parentSuite, name, declaredMode) { + constructor(parentSuite, name, declaredMode, comment) { this.parentSuite = parentSuite; this.name = name; this.fullName = (parentSuite ? parentSuite.fullName + ' ' + name : name).trim(); this.declaredMode = declaredMode; + this.comment = comment; /** @type {!Array<(!Test|!Suite)>} */ this.children = []; @@ -283,32 +285,40 @@ class TestRunner extends EventEmitter { } // bind methods so that they can be used as a DSL. - this.describe = this._addSuite.bind(this, TestMode.Run); - this.fdescribe = this._addSuite.bind(this, TestMode.Focus); - this.xdescribe = this._addSuite.bind(this, TestMode.Skip); - this.it = this._addTest.bind(this, TestMode.Run); - this.fit = this._addTest.bind(this, TestMode.Focus); - this.xit = this._addTest.bind(this, TestMode.Skip); + this.describe = this._addSuite.bind(this, TestMode.Run, ''); + this.fdescribe = this._addSuite.bind(this, TestMode.Focus, ''); + this.xdescribe = this._addSuite.bind(this, TestMode.Skip, ''); + this.it = this._addTest.bind(this, TestMode.Run, ''); + this.fit = this._addTest.bind(this, TestMode.Focus, ''); + this.xit = this._addTest.bind(this, TestMode.Skip, ''); this.beforeAll = this._addHook.bind(this, 'beforeAll'); this.beforeEach = this._addHook.bind(this, 'beforeEach'); this.afterAll = this._addHook.bind(this, 'afterAll'); this.afterEach = this._addHook.bind(this, 'afterEach'); } - _addTest(mode, name, callback) { + addTestDSL(dslName, mode, comment) { + this[dslName] = this._addTest.bind(this, mode, comment); + } + + addSuiteDSL(dslName, mode, comment) { + this[dslName] = this._addSuite.bind(this, mode, comment); + } + + _addTest(mode, comment, name, callback) { let suite = this._currentSuite; let isSkipped = suite.declaredMode === TestMode.Skip; while ((suite = suite.parentSuite)) isSkipped |= suite.declaredMode === TestMode.Skip; - const test = new Test(this._currentSuite, name, callback, isSkipped ? TestMode.Skip : mode, this._timeout); + const test = new Test(this._currentSuite, name, callback, isSkipped ? TestMode.Skip : mode, this._timeout, comment); this._currentSuite.children.push(test); this._tests.push(test); this._hasFocusedTestsOrSuites = this._hasFocusedTestsOrSuites || mode === TestMode.Focus; } - _addSuite(mode, name, callback) { + _addSuite(mode, comment, name, callback) { const oldSuite = this._currentSuite; - const suite = new Suite(this._currentSuite, name, mode); + const suite = new Suite(this._currentSuite, name, mode, comment); this._currentSuite.children.push(suite); this._currentSuite = suite; callback();