Add ExpectedToFail test extension (#551 / #676)

README.md

@@ -120,6 +120,7 @@ The least we can do is to thank them and list some of their accomplishments here
 #### 2022
 
 * [Filip Hrisafov](https://github.com/filiphr) contributed the [JSON Argument Source](https://junit-pioneer.org/docs/json-argument-source/) support (#101 / #492)
+* [Marcono1234](https://github.com/Marcono1234) contributed the [`@ExpectedToFail` extension](https://junit-pioneer.org/docs/expected-to-fail-tests/) (#551 / #676)
 * [Mathieu Fortin](https://github.com/mathieufortin01) contributed the `suspendForMs` attribute in [retrying tests](https://junit-pioneer.org/docs/retrying-test/) (#407 / #604)
 * [Pankaj Kumar](https://github.com/p1729) contributed towards improving GitHub actions (#587 / #611)
 * [Rob Spoor](https://github.com/robtimus) enabled non-static factory methods for `@CartesianTest.MethodFactory` (#628)

docs/docs-nav.yml

@@ -16,6 +16,8 @@
         url: /docs/disable-if-test-fails/
       - title: "Disable Parameterized Tests"
         url: /docs/disable-parameterized-tests/
+      - title: "Expected to Fail Tests"
+        url: /docs/expected-to-fail-tests/
       - title: "Issue information"
         url: /docs/issue/
       - title: "JSON Argument Source"

docs/expected-to-fail-tests.adoc

@@ -0,0 +1,43 @@
+:page-title: Expected to Fail Tests
+:page-description: Extends JUnit Jupiter with `@ExpectedToFail`, which marks a test method as temporarily 'expected to fail'
+:xp-demo-dir: ../src/demo/java
+:demo: {xp-demo-dir}/org/junitpioneer/jupiter/ExpectedToFailExtensionDemo.java
+
+Often tests fail due to a bug in the tested application or in used dependencies.
+Traditionally such a test method would be annotated with JUnit's `@Disabled`.
+However, this has the following disadvantages when the bug causing the test failure is fixed:
+
+* the developer might not notice the existing test method and create a new one
+* the existing test method might not be noticed and remains disabled for a long time after the bug has been fixed, adding no value for the project
+
+`@ExpectedToFail` tries to solve these issues.
+Unlike `@Disabled` it still executes the annotated test method but aborts the test if a test failure or error occurs.
+However, if the test is executed successfully the `@ExpectedToFail` annotation will cause a test failure because the test _is working_.
+This lets the developer know that they have fixed the bug (possibly by accident) and that they can now remove the `@ExpectedToFail` annotation from the test method.
+
+The annotation can only be used on methods and as meta-annotation on other annotation types.
+Similar to `@Disabled`, it has to be used in addition to a "testable" annotation, such as `@Test`.
+Otherwise the annotation has no effect.
+
+IMPORTANT: This annotation is *not* intended as a way to mark test methods which intentionally cause exceptions.
+Such test methods should use https://junit.org/junit5/docs/current/api/org.junit.jupiter.api/org/junit/jupiter/api/Assertions.html#assertThrows(java.lang.Class,org.junit.jupiter.api.function.Executable)[JUnit's `assertThrows`] or similar means to explicitly test for a specific exception class being thrown by a specific action.
+
+== Basic Use
+
+The test is aborted because the tested method `brokenMethod()` returns an incorrect result.
+
+[source,java,indent=0]
+----
+include::{demo}[tag=expected_to_fail]
+----
+
+A custom message can be provided, explaining why the tested code is not working as intended at the moment.
+
+[source,java,indent=0]
+----
+include::{demo}[tag=expected_to_fail_message]
+----
+
+== Thread-Safety
+
+This extension is thread-safe.

src/demo/java/org/junitpioneer/jupiter/ExpectedToFailExtensionDemo.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2016-2022 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junitpioneer.jupiter;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.junit.jupiter.api.Test;
+
+public class ExpectedToFailExtensionDemo {
+
+	// tag::expected_to_fail[]
+	@ExpectedToFail
+	@Test
+	void test() {
+		int actual = brokenMethod();
+		assertEquals(10, actual);
+	}
+	// end::expected_to_fail[]
+
+	// tag::expected_to_fail_message[]
+	@ExpectedToFail("Implementation bug in brokenMethod()")
+	@Test
+	void doSomething() {
+		int actual = brokenMethod();
+		assertEquals(10, actual);
+	}
+	// end::expected_to_fail_message[]
+
+	private int brokenMethod() {
+		return 0;
+	}
+
+}

src/main/java/org/junitpioneer/jupiter/ExpectedToFail.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2016-2022 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junitpioneer.jupiter;
+
+import static java.lang.annotation.ElementType.ANNOTATION_TYPE;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/**
+ * {@code @ExpectedToFail} is a JUnit Jupiter extension to mark test methods as temporarily
+ * 'expected to fail'.
+ * Such test methods will still be executed but when they result in a test failure or error
+ * the test will be aborted.
+ * However, if the test method unexpectedly executes successfully, it is marked as failure
+ * to let the developer know that the test is now successful and that the {@code @ExpectedToFail}
+ * annotation can be removed.
+ *
+ * <p>The big difference compared to JUnit's {@link org.junit.jupiter.api.Disabled @Disabled}
+ * annotation is that the developer is informed as soon as a test is successful again.
+ * This helps avoiding writing duplicate tests by accident and counteracts the accumulation
+ * of disabled tests over time.
+ *
+ * <p>The annotation can only be used on methods and as meta-annotation on other annotation types.
+ * Similar to {@code @Disabled}, it has to be used in addition to a "testable" annotation, such
+ * as {@link org.junit.jupiter.api.Test @Test}. Otherwise the annotation has no effect.
+ *
+ * <p><b>Important:</b> This annotation is <b>not</b> intended as a way to mark test methods
+ * which intentionally cause exceptions.
+ * Such test methods should use {@link org.junit.jupiter.api.Assertions#assertThrows(Class, org.junit.jupiter.api.function.Executable) assertThrows}
+ * or similar means to explicitly test for a specific exception class being thrown by a
+ * specific action.
+ *
+ * <p>For more details and examples, see
+ * <a href="https://junit-pioneer.org/docs/expected-to-fail-tests/" target="_top">the documentation on <code>@ExpectedToFail</code></a>.
+ * </p>
+ *
+ * @see org.junit.jupiter.api.Disabled
+ */
+/*
+ * Implementation note:
+ * Only supports METHOD and ANNOTATION_TYPE as targets but not test classes because there
+ * it is not clear what the 'correct' behavior would be when only a few test methods
+ * execute successfully. Would the developer then have to remove the @ExpectedToFail annotation
+ * from the test class and annotate methods individually?
+ */
+@Documented
+@Retention(RUNTIME)
+@Target({ METHOD, ANNOTATION_TYPE })
+@ExtendWith(ExpectedToFailExtension.class)
+public @interface ExpectedToFail {
+
+	/**
+	 * Defines the message to show when a test is aborted because it is failing.
+	 * This can be used for example to briefly explain why the tested code is not working
+	 * as intended at the moment.
+	 * An empty string (the default) causes a generic default message to be used.
+	 */
+	String value() default "";
+
+}

src/main/java/org/junitpioneer/jupiter/ExpectedToFailExtension.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2016-2022 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junitpioneer.jupiter;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.lang.reflect.Method;
+
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.InvocationInterceptor;
+import org.junit.jupiter.api.extension.ReflectiveInvocationContext;
+import org.junit.platform.commons.support.AnnotationSupport;
+import org.opentest4j.TestAbortedException;
+
+class ExpectedToFailExtension implements Extension, InvocationInterceptor {
+
+	/**
+	 * No-arg constructor for JUnit to be able to create an instance.
+	 */
+	public ExpectedToFailExtension() {
+	}
+
+	private static ExpectedToFail getExpectedToFailAnnotation(ExtensionContext context) {
+		return AnnotationSupport
+				.findAnnotation(context.getRequiredTestMethod(), ExpectedToFail.class)
+				.orElseThrow(() -> new IllegalStateException("@ExpectedToFail is missing."));
+
+	}
+
+	/**
+	 * Returns whether the exception should be preserved and reported as is instead
+	 * of considering it an 'expected to fail' exception.
+	 *
+	 * <p>This method is used for exceptions which abort test execution and should
+	 * have higher precedence than aborted exceptions thrown by this extension.
+	 */
+	private static boolean shouldPreserveException(Throwable t) {
+		// Note: Ideally would use the same logic JUnit uses to determine if exception is aborting
+		// execution, see its class OpenTest4JAndJUnit4AwareThrowableCollector
+		return TestAbortedException.class.isInstance(t);
+	}
+
+	private static <T> T invokeAndInvertResult(Invocation<T> invocation, ExtensionContext extensionContext)
+			throws Throwable {
+		try {
+			invocation.proceed();
+			// if no exception was thrown fall through and call fail(...) eventually
+		}
+		catch (Throwable t) {
+			if (shouldPreserveException(t)) {
+				throw t;
+			}
+
+			ExpectedToFail annotation = getExpectedToFailAnnotation(extensionContext);
+
+			String message = annotation.value();
+			if (message.isEmpty()) {
+				message = "Test marked as temporarily 'expected to fail' failed as expected";
+			}
+
+			throw new TestAbortedException(message, t);
+		}
+
+		return fail("Test marked as 'expected to fail' succeeded; remove @ExpectedToFail from it");
+	}
+
+	@Override
+	public void interceptTestMethod(Invocation<Void> invocation, ReflectiveInvocationContext<Method> invocationContext,
+			ExtensionContext extensionContext) throws Throwable {
+		invokeAndInvertResult(invocation, extensionContext);
+	}
+
+}

src/main/java/org/junitpioneer/jupiter/package-info.java

@@ -8,6 +8,7 @@
  *     <li>{@link org.junitpioneer.jupiter.DefaultLocale} and {@link org.junitpioneer.jupiter.DefaultTimeZone}</li>
  *     <li>{@link org.junitpioneer.jupiter.DisabledUntil}</li>
  *     <li>{@link org.junitpioneer.jupiter.DisableIfTestFails}</li>
+ *     <li>{@link org.junitpioneer.jupiter.ExpectedToFail}</li>
  *     <li>{@link org.junitpioneer.jupiter.ReportEntry}</li>
  *     <li>{@link org.junitpioneer.jupiter.RetryingTest}</li>
  *     <li>{@link org.junitpioneer.jupiter.StdIo}</li>