Skip to content

Commit

Permalink
Issue/fix gradle filter (#3257)
Browse files Browse the repository at this point in the history 
* Fix gradle filter integration within kotest runner

* add documentation to the functions for future contributors

* use specId instead of kclass

* fixes number of tests expectation in DiscoveryTest.kt
  • Loading branch information
@myuwono
myuwono committed Oct 23, 2022
1 parent d78332c commit 5db3e46
Show file tree
Hide file tree
Showing 6 changed files with 149 additions and 206 deletions.
8 changes: 4 additions & 4 deletions ...ework/kotest-framework-api/src/commonMain/kotlin/io/kotest/core/descriptors/descriptor.kt
View file
Expand Up @@ -148,9 +148,9 @@ sealed interface Descriptor {
fun getTreePrefix(): List<Descriptor> {
val ret = mutableListOf<Descriptor>()
var x = this
loop@ while(true) {
loop@ while (true) {
ret.add(0, x)
when(x) {
when (x) {
is SpecDescriptor -> {
break@loop
}
Expand Down Expand Up @@ -179,7 +179,7 @@ data class DescriptorId(
/**
* Treats the lhs and rhs both as wildcard regex one by one and check if it matches the other
*/
fun wildCardMatch(id: DescriptorId) :Boolean {
fun wildCardMatch(id: DescriptorId): Boolean {
val thisRegex = with(this.value) {
("\\Q$this\\E").replace("*", "\\E.*\\Q").toRegex()
}
Expand All @@ -204,7 +204,7 @@ fun Descriptor.append(name: String): TestDescriptor =
* This may be the same descriptor that this method is invoked on, if that descriptor
* is a root test.
*/
fun TestDescriptor.root(): TestDescriptor {
tailrec fun TestDescriptor.root(): TestDescriptor {
return when (parent) {
is SpecDescriptor -> this // if my parent is a spec, then I am a root
is TestDescriptor -> parent.root()
Expand Down
128 changes: 72 additions & 56 deletions ...jvmMain/kotlin/io/kotest/runner/junit/platform/gradle/GradleClassMethodRegexTestFilter.kt
View file
@@ -1,6 +1,7 @@
package io.kotest.runner.junit.platform.gradle

import io.kotest.core.descriptors.Descriptor
import io.kotest.core.descriptors.TestPath
import io.kotest.core.filter.TestFilter
import io.kotest.core.filter.TestFilterResult
import io.kotest.mpp.Logger
Expand All @@ -13,70 +14,85 @@ class GradleClassMethodRegexTestFilter(private val patterns: List<String>) : Tes
logger.log { Pair(descriptor.toString(), "Testing against $patterns") }
return when {
patterns.isEmpty() -> TestFilterResult.Include
patterns.any { match(it, descriptor) } -> TestFilterResult.Include
patterns.all { match(it, descriptor) } -> TestFilterResult.Include
else -> TestFilterResult.Exclude(null)
}
}

/**
* Matches the pattern supplied from gradle build script or command line interface.
*
* supports:
* - gradle test --tests "SomeTest"
* - gradle test --tests "*Test"
* - gradle test --tests "io.package.*"
* - gradle test --tests "io.package"
* - gradle test --tests "io.package.SomeTest"
* - gradle test --tests "io.package.SomeTest.first level context*"
* - gradle test --tests "io.package.SomeTest.*"
* - gradle test --tests "io.*.SomeTest"
* - gradle test --tests "SomeTest.first level context*"
* - gradle test --tests "*.first level context*"
*
* Exact nested context / test matching is NOT CURRENTLY SUPPORTED.
* Kotest support lazy test registration within nested context. Gradle test filter does not
* natively work nicely with kotest. In order to make it work we need to think of a way to
* recursively apply partial context-search as we dive deeper into the contexts.
*
* Notes to Maintainers:
*
* Gradle supplies a pattern string which corresponds to a well-formed regex object.
* This can be directly usable for kotest.
* - A* becomes \QA\E.*
* - A*Test becomes \QA\E.*\QTest\E
* - io.*.A*Test becomes \Qio.\E.*\Q.A\E.*\QTest\E
* - io.*.A*Test.AccountDetails* becomes \Qio.\E.*\Q.A\E.*\QTest.AccountDetails\E.*
* - io.*.A*Test.some test context* becomes \Qio.\E.*\Q.A\E.*\QTest.some test context\E.*
*/
private fun match(pattern: String, descriptor: Descriptor): Boolean {
val (prefixWildcard, pck, classname, path) = GradleTestPattern.parse(pattern)
return when (descriptor) {
is Descriptor.TestDescriptor -> when (path) {
null -> true
else -> descriptor.path(false).value.startsWith(path)
}

is Descriptor.SpecDescriptor -> when {
pck != null && classname != null && prefixWildcard -> descriptor.kclass.qualifiedName?.contains("$pck.$classname") ?: false
pck != null && classname != null -> descriptor.kclass.qualifiedName == "$pck.$classname"
pck != null && prefixWildcard -> descriptor.kclass.qualifiedName?.contains(pck) ?: true
pck != null -> descriptor.kclass.qualifiedName?.startsWith(pck) ?: true
classname != null && prefixWildcard -> descriptor.kclass.simpleName?.contains(classname) ?: false
classname != null -> descriptor.kclass.simpleName == classname
else -> true
}
val path = descriptor.dotSeparatedFullPath().value
val regexPattern = "^(.*)$pattern".toRegex() // matches pattern exactly
val laxRegexPattern = "^(.*)$pattern(.*)\$".toRegex() // matches pattern that can be followed by others
val packagePath = descriptor.spec().id.value.split(".").dropLast(1).joinToString(".") // io.kotest

val isSimpleClassMatch by lazy {
// SomeTest or *Test
descriptor.spec().id.value.split(".").lastOrNull()?.matches(pattern.toRegex()) ?: false
}
val isSpecMatched by lazy { descriptor.spec().id.value.matches(regexPattern) } // *.SomeTest
val isFullPathMatched by lazy { path.matches(regexPattern) } // io.*.SomeTest
val isFullPathDotMatched by lazy { "$path.".matches(regexPattern) } // io.*. or io.*.SomeTest.*

// if there's no uppercase in the supplied pattern, activate trigger relaxed matching
val doesNotContainUppercase by lazy { pattern.replace("\\Q", "").replace("\\E", "").all { !it.isUpperCase() } }

val isPackageMatched by lazy { doesNotContainUppercase && packagePath.matches(laxRegexPattern) } // io.kotest
val isPackageWithDotMatched by lazy { doesNotContainUppercase && "$packagePath.".matches(laxRegexPattern) } // io.kotest.*

return isSimpleClassMatch ||
isFullPathMatched ||
isFullPathDotMatched ||
isSpecMatched ||
isPackageMatched ||
isPackageWithDotMatched
}
}

data class GradleTestPattern(
val prefixWildcard: Boolean,
val pckage: String?,
val classname: String?,
val path: String?,
) {
companion object {

// if the regex starts with a lower case character, then we assume it is in the format package.Class.testpath
// otherwise, we assume it is in the format Class.testpath
// the .testpath is always optional, and at least Class or package must be specified
// additionally, patterns can start with a * in which case the pattern matches suffixes
fun parse(pattern: String): GradleTestPattern {
require(pattern.isNotBlank())

val prefixWildcard = pattern.startsWith("*")
val pattern2 = pattern
.removePrefix("*")
.removePrefix(".")
.replace("\\Q", "") // Quote start regex, added by Gradle
.replace("\\E", "") // Quote end regex, added by Gradle

val tokens = pattern2.split('.')

// Package names shouldn't contain any upper-case letters
val classIndex = tokens.indexOfFirst { token -> token.any { it.isUpperCase() } }

// if class is not specified, then we assume the entire string is a package
if (classIndex == -1) return GradleTestPattern(prefixWildcard, pattern2, null, null)

// if the class is the first part, then no package is specified
val pck = if (classIndex == 0) null else tokens.take(classIndex).joinToString(".")

val pathParts = tokens.drop(classIndex + 1)
val path = if (pathParts.isEmpty()) null else pathParts.joinToString(".")

return GradleTestPattern(prefixWildcard, pck, tokens[classIndex], path)
/**
* Returns a gradle-compatible dot-separated full path of the given descriptor.
* i.e. io.package.MyTest.given something -- should do something
*
* Note: I'm forced to do this... :(
*
* We cannot use the / separator for contexts as gradle rejects that.
* Filters also seemingly only works on first "." after the class. This was severely limiting.
* The other problem is that also means we can't have "." in the test / context path because gradle doesn't
* like it and will not even give us any candidate classes.
*/
private fun Descriptor.dotSeparatedFullPath(): TestPath = when (this) {
is Descriptor.SpecDescriptor -> TestPath(this.id.value)
is Descriptor.TestDescriptor -> when (this.parent) {
is Descriptor.SpecDescriptor -> TestPath("${this.parent.id.value}.${this.id.value}")
is Descriptor.TestDescriptor -> TestPath("${this.parent.dotSeparatedFullPath().value} -- ${this.id.value}")
}

}
}
15 changes: 14 additions & 1 deletion ...mMain/kotlin/io/kotest/runner/junit/platform/gradle/GradlePostDiscoveryFilterExtractor.kt
View file
Expand Up @@ -32,10 +32,17 @@ object GradlePostDiscoveryFilterExtractor {
val matcher = testMatcher(filter)
logger.log { Pair(null, "TestMatcher [$matcher]") }

val buildScriptIncludePatterns = buildScriptIncludePatterns(matcher)
logger.log { Pair(null, "buildScriptIncludePatterns [$buildScriptIncludePatterns]") }

val commandLineIncludePatterns = commandLineIncludePatterns(matcher)
logger.log { Pair(null, "commandLineIncludePatterns [$commandLineIncludePatterns]") }

val regexes = commandLineIncludePatterns.map { pattern(it) }
val regexes = buildList {
addAll(buildScriptIncludePatterns)
addAll(commandLineIncludePatterns)
}.map { pattern(it) }

logger.log { Pair(null, "ClassMethodNameFilter regexes [$regexes]") }
regexes
}.getOrElse { emptyList() }
Expand All @@ -52,6 +59,12 @@ object GradlePostDiscoveryFilterExtractor {
return field.get(obj) as List<Any>
}

private fun buildScriptIncludePatterns(obj: Any): List<Any> {
val field = obj::class.java.getDeclaredField("buildScriptIncludePatterns")
field.isAccessible = true
return field.get(obj) as List<Any>
}

private fun pattern(obj: Any): String {
val field = obj::class.java.getDeclaredField("pattern")
field.isAccessible = true
Expand Down
2 changes: 1 addition & 1 deletion ...otest-runner-junit5/src/jvmTest/kotlin/com/sksamuel/kotest/runner/junit5/DiscoveryTest.kt
View file
Expand Up @@ -61,7 +61,7 @@ class DiscoveryTest : FunSpec({
.build()
val engine = KotestJunitPlatformTestEngine()
val descriptor = engine.discover(req, UniqueId.forEngine("testengine"))
descriptor.classes.size shouldBe 27
descriptor.classes.size shouldBe 26
}

test("kotest should return classes if request has no included or excluded test engines") {
Expand Down
112 changes: 58 additions & 54 deletions ...est/kotlin/io/kotest/runner/junit/platform/gradle/GradleClassMethodRegexTestFilterTest.kt
View file
Expand Up @@ -9,33 +9,31 @@ import io.kotest.matchers.shouldBe

class GradleClassMethodRegexTestFilterTest : FunSpec({

test("include classes") {

context("include classes") {
val spec = GradleClassMethodRegexTestFilterTest::class.toDescriptor()
withData(
nameFn = { filters -> "should be INCLUDED when evaluating $filters" },
listOf("\\QGradleClassMethodRegexTestFilterTest\\E"),
listOf(".*\\QthodRegexTestFilterTest\\E"),
listOf(".*\\QTest\\E"),
listOf("\\Qio.kotest.runner.junit.platform.gradle.GradleClassMethodRegexTestFilterTest\\E"),
listOf(".*\\Q.platform.gradle.GradleClassMethodRegexTestFilterTest\\E"),
listOf(".*\\Qorm.gradle.GradleClassMethodRegexTestFilterTest\\E")
) { filters ->
GradleClassMethodRegexTestFilter(filters).filter(spec) shouldBe TestFilterResult.Include
}
}

GradleClassMethodRegexTestFilter(listOf("GradleClassMethodRegexTestFilterTest")).filter(spec) shouldBe
TestFilterResult.Include

GradleClassMethodRegexTestFilter(listOf("*thodRegexTestFilterTest")).filter(spec) shouldBe
TestFilterResult.Include

GradleClassMethodRegexTestFilter(listOf("GradleClassMethodRegexTestFilterTest2")).filter(spec) shouldBe
TestFilterResult.Exclude(null)

GradleClassMethodRegexTestFilter(listOf("GradleClassMethodRegexTestFilterTes")).filter(spec) shouldBe
TestFilterResult.Exclude(null)

GradleClassMethodRegexTestFilter(listOf("io.kotest.runner.junit.platform.gradle.GradleClassMethodRegexTestFilterTest"))
.filter(spec) shouldBe TestFilterResult.Include

GradleClassMethodRegexTestFilter(listOf("*orm.gradle.GradleClassMethodRegexTestFilterTest"))
.filter(spec) shouldBe TestFilterResult.Include

GradleClassMethodRegexTestFilter(listOf("*.platform.gradle.GradleClassMethodRegexTestFilterTest"))
.filter(spec) shouldBe TestFilterResult.Include

GradleClassMethodRegexTestFilter(listOf("io.kotest.runner.junit.platform.GradleClassMethodRegexTestFilterTest"))
.filter(spec) shouldBe TestFilterResult.Exclude(null)
context("exclude classes") {
val spec = GradleClassMethodRegexTestFilterTest::class.toDescriptor()
withData(
nameFn = { filters -> "should be EXCLUDED when evaluating $filters" },
listOf("\\QGradleClassMethodRegexTestFilterTest2\\E"),
listOf("\\QGradleClassMethodRegexTestFilterTes\\E"),
listOf("\\Qio.kotest.runner.junit.platform.GradleClassMethodRegexTestFilterTest\\E")
) { filters ->
GradleClassMethodRegexTestFilter(filters).filter(spec) shouldBe TestFilterResult.Exclude(null)
}
}

context("include packages") {
Expand All @@ -44,55 +42,61 @@ class GradleClassMethodRegexTestFilterTest : FunSpec({
val container = spec.append("a context")
val test = container.append("nested test")

test("Exact match - includes the spec and tests within it") {
val filter = GradleClassMethodRegexTestFilter(listOf("io.kotest.runner.junit.platform.gradle"))

filter.filter(spec) shouldBe TestFilterResult.Include
filter.filter(test) shouldBe TestFilterResult.Include
withData(
nameFn = { filters -> "should be INCLUDED when evaluating $filters" },
listOf("\\Qio.kotest.runner.junit.platform.gradle\\E"),
listOf("\\Qio.kotest.runner.junit.platform.gradle.\\E.*"),
listOf(".*\\Qnner.junit.platform.gradle\\E"),
listOf(".*\\Qnner.junit.platform.gradle.\\E.*"),
listOf(".*\\Q.junit.platform.gradle\\E"),
listOf("\\Qio.kotest.runner.junit.platform.gra\\E.*"),
listOf("\\Qio.kotest.runner.junit\\E"),
) { filters ->
GradleClassMethodRegexTestFilter(filters).filter(spec) shouldBe TestFilterResult.Include
}

GradleClassMethodRegexTestFilter(listOf("*nner.junit.platform.gradle"))
.filter(spec) shouldBe TestFilterResult.Include

GradleClassMethodRegexTestFilter(listOf("*.junit.platform.gradle"))
.filter(spec) shouldBe TestFilterResult.Include

GradleClassMethodRegexTestFilter(listOf("io.kotest.runner.junit.platform.gra"))
.filter(spec) shouldBe TestFilterResult.Include

GradleClassMethodRegexTestFilter(listOf("io.kotest.runner.junit"))
.filter(spec) shouldBe TestFilterResult.Include
withData(
nameFn = { filters -> "should be EXCLUDED when evaluating $filters" },
listOf("\\Qio.kotest.runner.junit2\\E"),
listOf("\\Qio.kotest.runner.junit\\E", ".*\\QSpec\\E"),
) { filters ->
GradleClassMethodRegexTestFilter(filters).filter(spec) shouldBe TestFilterResult.Exclude(null)
}

GradleClassMethodRegexTestFilter(listOf("io.kotest.runner.junit2"))
.filter(spec) shouldBe TestFilterResult.Exclude(null)
withData(
nameFn = { filters -> "should be INCLUDED when container and test were evaluated using $filters" },
listOf("\\QGradleClassMethodRegexTestFilterTest.a context\\E.*"),
listOf(".*\\QTest\\E", "\\QGradleClassMethodRegex\\E.*\\Q.a context\\E.*"),
) { filters ->
GradleClassMethodRegexTestFilter(filters).filter(container) shouldBe TestFilterResult.Include
GradleClassMethodRegexTestFilter(filters).filter(test) shouldBe TestFilterResult.Include
}
}

context("includes with test paths") {

val spec = GradleClassMethodRegexTestFilterTest::class.toDescriptor()
val container = spec.append("a context")
val test = container.append("nested test")
val fqcn = GradleClassMethodRegexTestFilterTest::class.qualifiedName

val fqcn = "\\Q${GradleClassMethodRegexTestFilterTest::class.qualifiedName}\\E"

withData(
nameFn = { "should be INCLUDED when filter is: $it" },
"$fqcn",
"$fqcn.a context",
"*.gradle.GradleClassMethodRegexTestFilterTest.a context",
"*adle.GradleClassMethodRegexTestFilterTest.a context",
"$fqcn.a context -- nested test",
fqcn,
"$fqcn\\Q.a context\\E.*",
".*\\Q.gradle.GradleClassMethodRegexTestFilterTest.a context\\E.*",
".*\\Qadle.GradleClassMethodRegexTestFilterTest.a context\\E.*"
) { filter ->
GradleClassMethodRegexTestFilter(listOf(filter))
.filter(test) shouldBe TestFilterResult.Include
}

withData(
nameFn = { "should be EXCLUDED when filter is: $it" },
"$fqcn.a context2",
"$fqcn.nested test",
"$fqcn.a context -- nested test2",
"*sMethodRegexTestFilterTest.a context -- nested test2",
"$fqcn\\Q.a context2\\E",
"$fqcn\\Q.nested test\\E",
"$fqcn\\Q.a context.nested test2\\E",
".*\\QsMethodRegexTestFilterTest.a context -- nested test2\\Q",
) { filter ->
GradleClassMethodRegexTestFilter(listOf(filter))
.filter(test) shouldBe TestFilterResult.Exclude(null)
Expand Down

0 comments on commit 5db3e46

Please sign in to comment.