feat: add test.failing method

docs/GlobalAPI.md

@@ -878,3 +878,24 @@ const add = (a, b) => a + b;
 
 test.todo('add should be associative');
 ```
+
+### `test.failing(name, fn)`
+
+Also under the aliases: `it.failing(name, fn)`, `xtest.failing(name, fn)`, `xit.failing(name, fn)`, `it.skip.failing(name, fn)`, `it.only.failing(name, fn)`
+
+Use `test.failing` when you are writing a test and expecting it to fail. These tests will behave the other way normal tests do. If `failing` test will throw any errors then it will pass. If it does not throw it will fail.
+
+_Note_: You can use this type of tests i.e. when writing code in a BDD way. In that case the tests will not show up as failing until they pass. Then you can just remove the `failing` modifier to make them pass.
+
+Example:
+
+```js
+test.failing('it is not equal', () => {
+  expect(5).toBe(6); // this test will pass
+});
+
+test.failing('it is equal', () => {
+  expect(10).toBe(10); // this test will fail
+});
+```
+

e2e/__tests__/__snapshots__/testFailing.test.ts.snap

@@ -0,0 +1,81 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`works with all statuses 1`] = `
+"FAIL __tests__/statuses.test.js
+  ✓ passes
+  ✕ fails
+  ✓ failing failes = passes
+  ✕ failing passes = fails
+  ○ skipped skips
+  ○ skipped skipped failing 1
+  ○ skipped skipped failing 2
+  ✎ todo todo
+
+  ● fails
+
+    expect(received).toBe(expected) // Object.is equality
+
+    Expected: 101
+    Received: 10
+
+      11 |
+      12 | it('fails', () => {
+    > 13 |   expect(10).toBe(101);
+         |              ^
+      14 | });
+      15 |
+      16 | it.skip('skips', () => {
+
+      at Object.toBe (__tests__/statuses.test.js:13:14)
+
+  ● failing passes = fails
+
+    Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error."
+`;
+
+exports[`works with only mode 1`] = `
+"FAIL __tests__/worksWithOnlyMode.test.js
+  block with only, should pass
+    ✓ failing failes = passes, should pass
+    ○ skipped failing test
+    ○ skipped passing test
+  block with only, should fail
+    ✕ failing passes = fails, should fail
+    ○ skipped failing test
+    ○ skipped passing test
+
+  ● block with only, should fail › failing passes = fails, should fail
+
+    Failing test passed even though it was supposed to fail. Remove \`.failing\` to remove error."
+`;
+
+exports[`works with skip mode 1`] = `
+"FAIL __tests__/worksWithSkipMode.test.js
+  block with only, should pass
+    ✕ failing test
+    ✓ failing failes = passes
+    ○ skipped skipped failing failes = passes, should pass
+    ○ skipped passing test
+  block with only, should fail
+    ✓ passing test
+    ✓ failing passes = fails
+    ○ skipped failing passes = fails, should fail
+    ○ skipped failing test
+
+  ● block with only, should pass › failing test
+
+    expect(received).toBe(expected) // Object.is equality
+
+    Expected: 101
+    Received: 10
+
+      12 |
+      13 |   it('failing test', () => {
+    > 14 |     expect(10).toBe(101);
+         |                ^
+      15 |   });
+      16 |
+      17 |   it.skip('passing test', () => {
+
+      at Object.toBe (__tests__/worksWithSkipMode.test.js:14:16)"
+`;

e2e/__tests__/testFailing.test.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import * as path from 'path';
+import {extractSummary} from '../Utils';
+import runJest from '../runJest';
+const dir = path.resolve(__dirname, '../test-failing');
+
+test('works with all statuses', () => {
+  const result = runJest(dir, ['statuses.test.js']);
+  expect(result.exitCode).toBe(1);
+  const {rest} = extractSummary(result.stderr);
+  expect(rest).toMatchSnapshot();
+});
+
+test('works with only mode', () => {
+  const result = runJest(dir, ['worksWithOnlyMode.test.js']);
+  expect(result.exitCode).toBe(1);
+  const {rest} = extractSummary(result.stderr);
+  expect(rest).toMatchSnapshot();
+});
+
+test('works with skip mode', () => {
+  const result = runJest(dir, ['worksWithSkipMode.test.js']);
+  expect(result.exitCode).toBe(1);
+  const {rest} = extractSummary(result.stderr);
+  expect(rest).toMatchSnapshot();
+});

e2e/test-failing/__tests__/statuses.test.js

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+it('passes', () => {
+  expect(10).toBe(10);
+});
+
+it('fails', () => {
+  expect(10).toBe(101);
+});
+
+it.skip('skips', () => {
+  expect(10).toBe(101);
+});
+
+it.todo('todo');
+
+it.failing('failing failes = passes', () => {
+  expect(10).toBe(101);
+});
+
+it.skip.failing('skipped failing 1', () => {
+  expect(10).toBe(10);
+});
+
+it.skip.failing('skipped failing 2', () => {
+  expect(10).toBe(101);
+});
+
+it.failing('failing passes = fails', () => {
+  expect(10).toBe(10);
+});

e2e/test-failing/__tests__/worksWithOnlyMode.test.js

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+describe('block with only, should pass', () => {
+  it.only.failing('failing failes = passes, should pass', () => {
+    expect(10).toBe(101);
+  });
+
+  it('failing test', () => {
+    expect(10).toBe(101);
+  });
+
+  it('passing test', () => {
+    expect(10).toBe(10);
+  });
+});
+
+describe('block with only, should fail', () => {
+  it.only.failing('failing passes = fails, should fail', () => {
+    expect(10).toBe(10);
+  });
+
+  it('failing test', () => {
+    expect(10).toBe(101);
+  });
+
+  it('passing test', () => {
+    expect(10).toBe(10);
+  });
+});

e2e/test-failing/__tests__/worksWithSkipMode.test.js

@@ -0,0 +1,42 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+describe('block with only, should pass', () => {
+  it.skip.failing('skipped failing failes = passes, should pass', () => {
+    expect(10).toBe(101);
+  });
+
+  it('failing test', () => {
+    expect(10).toBe(101);
+  });
+
+  it.skip('passing test', () => {
+    expect(10).toBe(10);
+  });
+
+  it.failing('failing failes = passes', () => {
+    expect(10).toBe(101);
+  });
+});
+
+describe('block with only, should fail', () => {
+  it.skip.failing('failing passes = fails, should fail', () => {
+    expect(10).toBe(10);
+  });
+
+  it.skip('failing test', () => {
+    expect(10).toBe(101);
+  });
+
+  it('passing test', () => {
+    expect(10).toBe(10);
+  });
+
+  it.failing('failing passes = fails', () => {
+    expect(10).toBe(101);
+  });
+});

e2e/test-failing/package.json

@@ -0,0 +1,5 @@
+{
+  "jest": {
+    "testEnvironment": "node"
+  }
+}

packages/jest-circus/src/__tests__/circusItFailingTestError.test.ts

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type {Global} from '@jest/types';
+
+let circusIt: Global.It;
+let circusTest: Global.It;
+
+// using jest-jasmine2's 'it' to test jest-circus's 'it'. Had to differentiate
+// the two with this alias.
+
+const aliasCircusIt = () => {
+  const {it, test} = require('../');
+  circusIt = it;
+  circusTest = test;
+};
+
+aliasCircusIt();
+
+describe('test/it.failing error throwing', () => {
+  it("it doesn't throw an error with valid arguments", () => {
+    expect(() => {
+      circusIt.failing('test1', () => {});
+    }).not.toThrowError();
+  });
+  it('it throws error with missing callback function', () => {
+    expect(() => {
+      circusIt.failing('test2');
+    }).toThrowError(
+      'Missing second argument. It must be a callback function. Perhaps you want to use `test.todo` for a test placeholder.',
+    );
+  });
+  it("it throws an error when first argument isn't valid", () => {
+    expect(() => {
+      circusIt.failing(() => {});
+    }).toThrowError(
+      'Invalid first argument, () => {}. It must be a named class, named function, number, or string.',
+    );
+  });
+  it('it throws an error when callback function is not a function', () => {
+    expect(() => {
+      circusIt.failing('test4', 'test4b');
+    }).toThrowError(
+      'Invalid second argument, test4b. It must be a callback function.',
+    );
+  });
+  it('test throws error with missing callback function', () => {
+    expect(() => {
+      circusTest.failing('test5');
+    }).toThrowError(
+      'Missing second argument. It must be a callback function. Perhaps you want to use `test.todo` for a test placeholder.',
+    );
+  });
+  it("test throws an error when first argument isn't a string", () => {
+    expect(() => {
+      circusTest.failing(() => {});
+    }).toThrowError(
+      'Invalid first argument, () => {}. It must be a named class, named function, number, or string.',
+    );
+  });
+  it('test throws an error when callback function is not a function', () => {
+    expect(() => {
+      circusTest.failing('test7', 'test8b');
+    }).toThrowError(
+      'Invalid second argument, test8b. It must be a callback function.',
+    );
+  });
+});

packages/jest-circus/src/eventHandler.ts

@@ -122,7 +122,7 @@ const eventHandler: Circus.EventHandler = (event, state) => {
     }
     case 'add_test': {
       const {currentDescribeBlock, currentlyRunningTest, hasStarted} = state;
-      const {asyncError, fn, mode, testName: name, timeout} = event;
+      const {asyncError, fn, mode, testName: name, timeout, failing} = event;
 
       if (currentlyRunningTest) {
         currentlyRunningTest.errors.push(
@@ -147,6 +147,7 @@ const eventHandler: Circus.EventHandler = (event, state) => {
         currentDescribeBlock,
         timeout,
         asyncError,
+        failing,
       );
       if (currentDescribeBlock.mode !== 'skip' && test.mode === 'only') {
         state.hasFocusedTests = true;

packages/jest-circus/src/index.ts

@@ -129,6 +129,15 @@ const test: Global.It = (() => {
     timeout?: number,
   ): void => _addTest(testName, 'only', fn, test.only, timeout);
 
+  const bindFailing = (mode: Circus.TestMode) => {
+    const failing = (
+      testName: Circus.TestNameLike,
+      fn?: Circus.TestFn,
+      timeout?: number,
+    ): void => _addTest(testName, mode, fn, failing, timeout, true);
+    return failing;
+  };
+
   test.todo = (testName: Circus.TestNameLike, ...rest: Array<any>): void => {
     if (rest.length > 0 || typeof testName !== 'string') {
       throw new ErrorWithStack(
@@ -149,6 +158,7 @@ const test: Global.It = (() => {
       timeout?: number,
     ) => void,
     timeout?: number,
+    failing?: boolean,
   ) => {
     const asyncError = new ErrorWithStack(undefined, testFn);
 
@@ -173,6 +183,7 @@ const test: Global.It = (() => {
 
     return dispatchSync({
       asyncError,
+      failing: failing === undefined ? false : failing,
       fn,
       mode,
       name: 'add_test',
@@ -185,6 +196,10 @@ const test: Global.It = (() => {
   only.each = bindEach(only);
   skip.each = bindEach(skip);
 
+  only.failing = bindFailing('only');
+  skip.failing = bindFailing('skip');
+
+  test.failing = bindFailing();
   test.only = only;
   test.skip = skip;