[SUREFIRE-2040] No tests executed with junit-platform-suite and -Dtest=TestSuite

maven-surefire-plugin/src/site/apt/examples/junit-platform.apt.vm

@@ -294,6 +294,7 @@ Using JUnit 5 Platform
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
+                <version>${project.version}</version>
                 <dependencies>
                     <dependency>
                         <groupId>org.junit.jupiter</groupId>
@@ -334,6 +335,7 @@ Using JUnit 5 Platform
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
+                <version>${project.version}</version>
                 <dependencies>
                     <dependency>
                         <groupId>org.junit.vintage</groupId>
@@ -411,6 +413,41 @@ Using JUnit 5 Platform
 </dependencies>
 +---+
 
+** JUnit5 Suite
+
+  For more information see this
+  {{{https://github.com/apache/maven-surefire/tree/master/surefire-its/src/test/resources/junit5-suite}example with surefire integration test}}
+  and the {{{https://junit.org/junit5/docs/current/user-guide/#junit-platform-suite-engine}tutorial}}.
+
++---+
+<dependencies>
+    <dependency>
+        <groupId>org.junit.jupiter</groupId>
+        <artifactId>junit-jupiter-engine</artifactId>
+        <version>5.8.2</version>
+        <scope>test</scope>
+    </dependency>
+    <dependency>
+        <groupId>org.junit.platform</groupId>
+        <artifactId>junit-platform-suite-engine</artifactId>
+        <version>1.8.2</version>
+        <scope>test</scope>
+    </dependency>
+</dependencies>
+<build>
+    <plugins>
+        <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <version>${project.version}</version>
+            <configuration>
+                <test><!-- your test suite class name should be here --></test>
+            </configuration>
+        </plugin>
+    </plugins>
+</build>
++---+
+
 * Provider Selection
 
    If nothing is configured, Surefire detects which JUnit version to use by the following algorithm:

surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire1787JUnit5IT.java

@@ -22,6 +22,9 @@
 import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
 import org.junit.Test;
 
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
 /**
  *
  */
@@ -129,4 +132,17 @@ public void junit4Runner()
             .verifyTextInLog( "Running pkg.JUnit5Tests" )
             .verifyTextInLog( "Using auto detected provider org.apache.maven.surefire.junit4.JUnit4Provider" );
     }
+
+    @Test
+    public void junit5Suite() throws Exception
+    {
+        unpack( "junit5-suite" )
+            .executeTest()
+            .verifyErrorFree( 1 )
+            .verifyTextInLog(
+                "Using auto detected provider org.apache.maven.surefire.junitplatform.JUnitPlatformProvider" )
+            .verifyTextInLog( "Running pkg.JUnit5Test" )
+            .verifyTextInLog( "Running pkg.domain.AxTest" )
+            .assertThatLogLine( containsString( "Running pkg.domain.BxTest" ), equalTo( 0 ) );
+    }
 }

surefire-its/src/test/resources/junit5-suite/pom.xml

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.example</groupId>
+  <artifactId>junit5-suite</artifactId>
+  <version>1.0-SNAPSHOT</version>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <maven.compiler.source>${java.specification.version}</maven.compiler.source>
+    <maven.compiler.target>${java.specification.version}</maven.compiler.target>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <!--
+      The engine uses version 5.8.2. Due to the JUnit5 modularity model splits api and implementation,
+      the implementations should be backwards compatible with older versions of api, we are testing this corner case
+      for internal Surefire requirements. The user may or may not split the api and impl and versions.
+      -->
+      <version>5.8.0</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-suite-api</artifactId>
+      <!--
+      The engine uses version 5.8.2. Due to the JUnit5 modularity model splits api and implementation,
+      the implementations should be backwards compatible with old versions of api, we are testing this corner case
+      for internal Surefire requirements. The user may or may not split the api and impl and versions.
+      -->
+      <version>1.8.0</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-surefire-plugin</artifactId>
+          <version>${surefire.version}</version>
+          <configuration>
+            <test>JUnit5Tests</test>
+          </configuration>
+          <dependencies>
+            <!--
+            You may split the impl from api as follows. It avoids a situation where your tests have
+            direct access to the engine internals.
+            As an example, the impl:5.9.0 should be backwards with api:1.8.0 and one limitation where you
+            would not observe features introduced in 1.9.0 and 5.9.0.
+            -->
+            <dependency>
+              <groupId>org.junit.jupiter</groupId>
+              <artifactId>junit-jupiter-engine</artifactId>
+              <version>5.8.2</version>
+            </dependency>
+            <dependency>
+              <groupId>org.junit.platform</groupId>
+              <artifactId>junit-platform-suite</artifactId>
+              <version>1.8.2</version>
+            </dependency>
+          </dependencies>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+
+</project>

surefire-its/src/test/resources/junit5-suite/src/test/java/pkg/JUnit5Tests.java

@@ -0,0 +1,29 @@
+package pkg;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.platform.suite.api.SelectClasses;
+import org.junit.platform.suite.api.Suite;
+
+@Suite
+@SelectClasses({pkg.domain.AxTest.class})
+public class JUnit5Tests
+{
+}

surefire-its/src/test/resources/junit5-suite/src/test/java/pkg/domain/AxTest.java

@@ -0,0 +1,30 @@
+package pkg.domain;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.jupiter.api.Test;
+
+public class AxTest
+{
+    @Test
+    void test()
+    {
+    }
+}

surefire-its/src/test/resources/junit5-suite/src/test/java/pkg/domain/BxTest.java

@@ -0,0 +1,33 @@
+package pkg.domain;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+public class BxTest
+{
+    @Test
+    void test()
+    {
+        fail();
+    }
+}

surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProvider.java

@@ -32,6 +32,7 @@
 import static org.apache.maven.surefire.api.report.ConsoleOutputCapture.startCapture;
 import static org.apache.maven.surefire.api.report.RunMode.NORMAL_RUN;
 import static org.apache.maven.surefire.api.report.RunMode.RERUN_TEST_AFTER_FAILURE;
+import static org.apache.maven.surefire.api.testset.TestListResolver.optionallyWildcardFilter;
 import static org.apache.maven.surefire.api.util.TestsToRun.fromClass;
 import static org.apache.maven.surefire.shared.utils.StringUtils.isBlank;
 import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass;
@@ -55,7 +56,6 @@
 import org.apache.maven.surefire.api.report.ReporterException;
 import org.apache.maven.surefire.api.report.ReporterFactory;
 import org.apache.maven.surefire.api.suite.RunResult;
-import org.apache.maven.surefire.api.testset.TestListResolver;
 import org.apache.maven.surefire.api.testset.TestSetFailedException;
 import org.apache.maven.surefire.api.util.ScanResult;
 import org.apache.maven.surefire.api.util.SurefireReflectionException;
@@ -275,11 +275,11 @@ private Filter<?>[] newFilters()
                 .map( TagFilter::excludeTags )
                 .ifPresent( filters::add );
 
-        TestListResolver testListResolver = parameters.getTestRequest().getTestListResolver();
-        if ( !testListResolver.isEmpty() )
-        {
-            filters.add( new TestMethodFilter( testListResolver ) );
-        }
+        of( optionallyWildcardFilter( parameters.getTestRequest().getTestListResolver() ) )
+            .filter( f -> !f.isEmpty() )
+            .filter( f -> !f.isWildcard() )
+            .map( TestMethodFilter::new )
+            .ifPresent( filters::add );
 
         getPropertiesList( INCLUDE_JUNIT5_ENGINES_PROP )
             .map( EngineFilter::includeEngines )

surefire-providers/surefire-junit-platform/src/test/java/org/apache/maven/surefire/junitplatform/JUnitPlatformProviderTest.java

@@ -29,6 +29,7 @@
 import static org.apache.maven.surefire.api.report.RunMode.NORMAL_RUN;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -43,6 +44,7 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 import static org.mockito.Mockito.withSettings;
+import static org.powermock.reflect.Whitebox.getInternalState;
 
 import java.io.PrintStream;
 import java.util.ArrayList;
@@ -891,6 +893,65 @@ public void parsesConfigurationParameters()
         assertEquals( "EOF", provider.getConfigurationParameters().get( "qux" ) );
     }
 
+    @Test
+    public void shouldFilterTestMethod()
+    {
+        ProviderParameters providerParameters = providerParametersMock();
+        TestListResolver testListResolver = new TestListResolver( "**/*Test#test*" );
+        assertFalse( testListResolver.isEmpty() );
+        assertFalse( testListResolver.isWildcard() );
+        TestRequest request = new TestRequest( null, null, testListResolver, 0 );
+        when( providerParameters.getTestRequest() ).thenReturn( request );
+
+        JUnitPlatformProvider provider = new JUnitPlatformProvider( providerParameters );
+
+        assertThat( provider.getFilters() )
+            .hasSize( 1 );
+
+        assertThat( provider.getFilters()[0] )
+            .isInstanceOf( TestMethodFilter.class );
+
+        Object expectedTestListResolver = getInternalState( provider.getFilters()[0], "testListResolver" );
+
+        assertThat( expectedTestListResolver )
+            .isInstanceOf( TestListResolver.class );
+
+        assertThat( expectedTestListResolver  )
+            .isSameAs( testListResolver );
+    }
+
+    @Test
+    public void shouldNotFilterEmpty()
+    {
+        ProviderParameters providerParameters = providerParametersMock();
+        TestListResolver testListResolver = new TestListResolver( "" );
+        assertTrue( testListResolver.isEmpty() );
+        assertFalse( testListResolver.isWildcard() );
+        TestRequest request = new TestRequest( null, null, testListResolver, 0 );
+        when( providerParameters.getTestRequest() ).thenReturn( request );
+
+        JUnitPlatformProvider provider = new JUnitPlatformProvider( providerParameters );
+
+        assertThat( provider.getFilters() )
+            .isEmpty();
+    }
+
+    @Test
+    public void shouldNotFilterWildcard()
+    {
+        ProviderParameters providerParameters = providerParametersMock();
+        TestListResolver testListResolver = new TestListResolver( "*.java" );
+        assertTrue( testListResolver.isWildcard() );
+        assertFalse( testListResolver.isEmpty() );
+        TestRequest request = new TestRequest( null, null, testListResolver, 0 );
+        when( providerParameters.getTestRequest() ).thenReturn( request );
+
+        JUnitPlatformProvider provider = new JUnitPlatformProvider( providerParameters );
+
+        assertThat( provider.getFilters() )
+            .isEmpty();
+    }
+
     @Test
     public void executesSingleTestIncludedByName()
                     throws Exception