Disable parameterized test invocations by display name (#163 / #175)

Jupiter's `@Disabled`-based annotations disable entire test template
methods and can thus not be used to disable individual parameterized
test invocations. This extension adds that capability by matching
based on display name. Display names can be evaluated by searching
for substrings or matching against regular expressions (or both).

Because this extension belongs into the `params` package but also
needs to access `PioneerAnnotationUtils`, we needed to find a way
to make them accessible without making the class public. To prevent
duplication, we went with a (clever? abominable?) reflection hack.

Closes: #163
PR: #175

docs/disable-if-display-name.adoc

@@ -0,0 +1,76 @@
+:page-title: @DisableIfDisplayName
+:page-description: JUnit Jupiter extensions to selectively disable parameterized tests
+
+The `@DisableIfDisplayName` annotation can be used to selectively disable parameterized tests based on their display names, which are dynamically registered on runtime.
+The annotation is only supported on test method level for parameterized tests.
+Unlike the `@Disabled` API provided in JUnit Jupiter, which disables the test on first encounter of the annotation , `@DisableIfDisplayName` is validated before each parameterized test execution.
+As a consequence, instead of disabling the entire set of parameterized tests, each test (name) can be evaluated and possibly disabled individually.
+
+[source,java]
+----
+// disable invocations whose display name contains "disable"
+@DisableIfDisplayName(contains = "disable")
+@ParameterizedTest(name = "See if enabled with {0}")
+@ValueSource(
+        strings = {
+                "disable who", // ~> disabled
+                "you, disable you", // ~> disabled
+                "why am I disabled", // ~> disabled
+                "what has been disabled must stay disabled", // ~> disabled
+                "fine disable me all you want", // ~> disabled
+                "not those one, though!" // ~> NOT disabled
+        }
+)
+void testExecutionDisabled(String reason) {
+    if (reason.contains("disable"))
+        fail("Test should've been disabled " + reason);
+}
+----
+
+You can also specify more than one substring at a time:
+
+[source,java]
+----
+@DisableIfDisplayName(contains = {"1", "2"})
+@ParameterizedTest(name = "See if enabled with {0}")
+@ValueSource(ints = { 1, 2, 3, 4, 5 })
+void testDisplayNameString(int num) {
+    if (num == 1 || num == 2)
+        fail("Test should've been disabled for " + num);
+}
+----
+
+If substrings are not powerful enough, you can also use regular expressions:
+
+[source,java]
+----
+// disable invocations whose display name contains "disable " or "disabled "
+@DisableIfDisplayName(matches = ".*disabled?\\s.*")
+@ParameterizedTest(name = "See if enabled with {0}")
+@ValueSource(
+        strings = {
+                "disable who", // ~> disabled
+                "you, disable you", // ~> disabled
+                "why am I disabled", // ~> NOT disabled
+                "what has been disabled must stay disabled", // ~> disabled
+                "fine disable me all you want", // ~> disabled
+                "not those one, though!" // ~> NOT disabled
+        }
+)
+void single(String reason) {
+    // ...
+}
+----
+
+You can even use both, in which case a test is disabled if contains a substring _or_ matches an expression:
+
+[source,java]
+----
+@DisableIfDisplayName(contains = "000", matches = ".*10?" )
+@ParameterizedTest(name = "See if enabled with {0}")
+@ValueSource(ints = { 1, 10, 100, 1_000, 10_000 })
+void containsAndMatches_containsAndMatches(int number) {
+    if (number != 100)
+        fail("Test should've been disabled for " + number);
+}
+----

docs/docs-nav.yml

@@ -8,6 +8,8 @@
         url: /docs/environment-variables/
       - title: "@DefaultLocale and @DefaultTimeZone"
         url: /docs/default-locale-timezone/
+      - title: "@DisableIfDisplayName"
+        url: /docs/disable-if-display-name/
       - title: "Range Sources"
         url: /docs/range-sources/
       - title: "@RetryingTest"
@@ -20,3 +22,4 @@
         url: /docs/temp-directory/
       - title: "Vintage @Test"
         url: /docs/vintage-test/
+

src/main/java/org/junitpioneer/jupiter/params/DisableIfDisplayName.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2015-2020 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junitpioneer.jupiter.params;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.condition.DisabledIf;
+import org.junit.jupiter.api.extension.ExecutionCondition;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.params.ParameterizedTest;
+
+/**
+ * {@code @DisableIfDisplayName} is a JUnit Jupiter extension that can be used to
+ * selectively disable {@link ParameterizedTest} based on their
+ * {@link ExtensionContext#getDisplayName() display name}.
+ *
+ * <p>The extension is an {@link ExecutionCondition} that validates dynamically registered tests.
+ * Unlike {@link Disabled} or {@link DisabledIf} annotations, this extension doesn't disable
+ * the whole test method. With {@code DisableIfDisplayName}, it is possible to selectively disable
+ * tests out of the plethora of dynamically registered parameterized tests.</p>
+ *
+ * <p>If neither {@link DisableIfDisplayName#contains() contains} nor
+ * {@link DisableIfDisplayName#matches() matches} is configured, the extension will throw an exception.
+ * It is possible to configure both, in which case the test gets disabled if at least one substring
+ * was found <em>or</em> at least one regular expression matched.</p>
+ *
+ * @since 0.8
+ * @see DisableIfNameExtension
+ */
+@Target(ElementType.METHOD)
+@Retention(RetentionPolicy.RUNTIME)
+@ExtendWith(DisableIfNameExtension.class)
+public @interface DisableIfDisplayName {
+
+	/**
+	 * Disable test cases whose display name contain the specified strings
+	 * (according to {@link String#contains(CharSequence)}).
+	 *
+	 * @return test case display name substrings
+	 */
+	String[] contains() default {};
+
+	/**
+	 * Disable test cases whose display name matches the specified regular rxpression
+	 * (according to {@link String#matches(java.lang.String)}).
+	 *
+	 * @return test case display name regular expressions
+	 */
+	String[] matches() default {};
+
+}

src/main/java/org/junitpioneer/jupiter/params/DisableIfNameExtension.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2015-2020 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junitpioneer.jupiter.params;
+
+import static java.lang.String.format;
+import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled;
+import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled;
+import static org.junitpioneer.jupiter.params.PioneerAnnotationUtils.findClosestEnclosingAnnotation;
+
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.extension.ConditionEvaluationResult;
+import org.junit.jupiter.api.extension.ExecutionCondition;
+import org.junit.jupiter.api.extension.ExtensionConfigurationException;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+public class DisableIfNameExtension implements ExecutionCondition {
+
+	@Override
+	public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) {
+		/* We need to make sure not to accidentally disable the @ParameterizedTest method itself.
+		 * Since the Jupiter API offers no way to identify that case directly, we use a hack that relies
+		 * on the fact that the invocations' unique IDs end with a "test-template-invocation section."
+		 * The @ParameterizedTest-annotated method's own unique ID does not contain that string.
+		 */
+		if (!context.getUniqueId().contains("test-template-invocation"))
+			return enabled("Never disable parameterized test method itself");
+		return findClosestEnclosingAnnotation(context, DisableIfDisplayName.class)
+				.map(annotation -> disable(context, annotation))
+				.orElseGet(() -> enabled("No instructions to disable"));
+	}
+
+	private ConditionEvaluationResult disable(ExtensionContext context, DisableIfDisplayName disableInstruction) {
+		String[] substrings = disableInstruction.contains();
+		String[] regExps = disableInstruction.matches();
+		boolean checkSubstrings = substrings.length > 0;
+		boolean checkRegExps = regExps.length > 0;
+
+		if (!checkSubstrings && !checkRegExps)
+			throw new ExtensionConfigurationException(
+				"@DisableIfDisplayName requires that either `contains` or `matches` is specified, but both are empty.");
+
+		String displayName = context.getDisplayName();
+		ConditionEvaluationResult substringResults = disableIfContains(displayName, substrings);
+		ConditionEvaluationResult regExpResults = disableIfMatches(displayName, regExps);
+
+		if (checkSubstrings && checkRegExps)
+			return checkResults(substringResults, regExpResults);
+		if (checkSubstrings)
+			return substringResults;
+		if (checkRegExps)
+			return regExpResults;
+
+		// can't happen, all four combinations are covered
+		return null;
+	}
+
+	private ConditionEvaluationResult checkResults(ConditionEvaluationResult substringResults,
+			ConditionEvaluationResult regExpResults) {
+		boolean disabled = substringResults.isDisabled() || regExpResults.isDisabled();
+		String reason = format("%s %s", substringResults.getReason(), regExpResults.getReason());
+		return disabled ? disabled(reason) : enabled(reason);
+	}
+
+	private ConditionEvaluationResult disableIfMatches(String displayName, String[] regExps) {
+		//@formatter:off
+		String matches = Stream
+				.of(regExps)
+				.filter(displayName::matches)
+				.collect(Collectors.joining("', '"));
+		return matches.isEmpty()
+				? enabled("Display name '" + displayName + " doesn't match any regular expression.")
+				: disabled("Display name '" + displayName + "' matches '" + matches + "'.");
+		//@formatter:on
+	}
+
+	private ConditionEvaluationResult disableIfContains(String displayName, String[] substrings) {
+		//@formatter:off
+		String matches = Stream
+				.of(substrings)
+				.filter(displayName::contains)
+				.collect(Collectors.joining("', '"));
+		return matches.isEmpty()
+				? enabled("Display name '" + displayName + " doesn't contain any substring.")
+				: disabled("Display name '" + displayName + "' contains '" + matches + "'.");
+		//@formatter:on
+	}
+
+}

src/main/java/org/junitpioneer/jupiter/params/PioneerAnnotationUtils.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright 2015-2020 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junitpioneer.jupiter.params;
+
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
+import java.util.Optional;
+
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+/**
+ * Copy of {@code org.junitpioneer.jupiter.PioneerAnnotationUtils}.
+ *
+ * <p>This copy is necessary to keep all utils package-visible. In order not to duplicate
+ * a lot of code, which adds the challenge to keep the implementations tested and in sync,
+ * we use reflection to access the package-visible original implementation.</p>
+ */
+class PioneerAnnotationUtils {
+
+	private static final Class<?> PIONEER_ANNOTATION_UTILS;
+	private static final Method FIND_CLOSEST_ENCLOSING_ANNOTATION;
+
+	static {
+		try {
+			PIONEER_ANNOTATION_UTILS = Class.forName("org.junitpioneer.jupiter.PioneerAnnotationUtils");
+			FIND_CLOSEST_ENCLOSING_ANNOTATION = PIONEER_ANNOTATION_UTILS
+					.getMethod("findClosestEnclosingAnnotation", ExtensionContext.class, Class.class);
+			FIND_CLOSEST_ENCLOSING_ANNOTATION.setAccessible(true);
+		}
+		catch (ReflectiveOperationException ex) {
+			throw new RuntimeException("Pioneer could not initialize itself.", ex);
+		}
+	}
+
+	private PioneerAnnotationUtils() {
+		// private constructor to prevent instantiation of utility class
+	}
+
+	@SuppressWarnings("unchecked")
+	public static <A extends Annotation> Optional<A> findClosestEnclosingAnnotation(ExtensionContext context,
+			Class<A> annotationType) {
+		try {
+			return (Optional<A>) FIND_CLOSEST_ENCLOSING_ANNOTATION.invoke(null, context, annotationType);
+		}
+		catch (ReflectiveOperationException ex) {
+			throw new RuntimeException("Internal Pioneer error.", ex);
+		}
+	}
+
+}