Skip to content

Commit

Permalink
Fixed instrumentation counter in IntelliJ verifier
Browse files Browse the repository at this point in the history
- IntelliJ Engine version upgraded to 1.0.680
- added tests on verification error

Fixes #210, #211,  #212
  • Loading branch information
shanshin committed Aug 23, 2022
1 parent 5e50881 commit f7f0d1c
Show file tree
Hide file tree
Showing 10 changed files with 404 additions and 96 deletions.
Expand Up @@ -38,15 +38,15 @@ internal class VerificationTests : BaseGradleScriptTest() {
}

@Test
fun testNotVerifiedIntelliJ() {
val build = diverseBuild(languages = ALL_LANGUAGES, engines = listOf(CoverageEngineVendor.INTELLIJ))
fun testVerificationError() {
val build = diverseBuild(languages = ALL_LANGUAGES, engines = ALL_ENGINES)
build.addKoverRootProject {
sourcesFrom("simple")
sourcesFrom("verification")

kover {
verify {
rule {
name = "test rule"
name = "counts rule"
bound {
minValue = 58
maxValue = 60
Expand All @@ -57,55 +57,94 @@ internal class VerificationTests : BaseGradleScriptTest() {
maxValue = 3
}
}
}
}
}

build.prepare().runWithError("koverVerify") {
output {
assertTrue {
this.contains(
"> Rule 'test rule' violated:\n" +
" covered lines percentage is 57.142900, but expected minimum is 58\n" +
" covered lines count is 4, but expected maximum is 3"
)
}
}
}
}

@Test
fun testNotVerifiedJaCoCo() {
val build = diverseBuild(languages = ALL_LANGUAGES, engines = listOf(CoverageEngineVendor.JACOCO))
build.addKoverRootProject {
sourcesFrom("simple")

kover {
verify {
rule {
name = "test rule"
name = "fully uncovered instructions by classes"
target = VerificationTarget.CLASS
bound {
minValue = 58
maxValue = 60
counter = CounterType.INSTRUCTION
valueType = VerificationValueType.MISSED_PERCENTAGE
minValue = 100
}
}
rule {
name = "fully covered instructions by packages"
target = VerificationTarget.PACKAGE
bound {
counter = CounterType.INSTRUCTION
valueType = VerificationValueType.COVERED_PERCENTAGE
minValue = 100
}
}
rule {
name = "branches by classes"
target = VerificationTarget.CLASS
bound {
counter = CounterType.BRANCH
valueType = VerificationValueType.COVERED_COUNT
minValue = 2
maxValue = 3
minValue = 1000
}
}
rule {
name = "missed packages"
target = VerificationTarget.PACKAGE
bound {
valueType = VerificationValueType.MISSED_COUNT
maxValue = 1
}
}
}
}
}

build.prepare().runWithError("koverVerify") {
output {
assertTrue {
this.contains(
"[ant:jacocoReport] Rule violated for bundle :: lines covered ratio is 0.50, but expected minimum is 0.58\n" +
"[ant:jacocoReport] Rule violated for bundle :: lines covered count is 4, but expected maximum is 3"
)
}
build.prepare().runWithError("koverHtmlReport", "koverVerify") {
verification {
assertIntelliJResult("""Rule 'counts rule' violated:
lines covered percentage is 46.590900, but expected minimum is 58
lines covered count is 41, but expected maximum is 3
Rule 'fully uncovered instructions by classes' violated:
instructions missed percentage for class 'org.jetbrains.kover.test.functional.verification.FullyCovered' is 0.000000, but expected minimum is 100
instructions missed percentage for class 'org.jetbrains.kover.test.functional.verification.PartiallyCoveredFirst' is 44.642900, but expected minimum is 100
instructions missed percentage for class 'org.jetbrains.kover.test.functional.verification.PartiallyCoveredSecond' is 51.666700, but expected minimum is 100
instructions missed percentage for class 'org.jetbrains.kover.test.functional.verification.subpackage.SubFullyCovered' is 0.000000, but expected minimum is 100
instructions missed percentage for class 'org.jetbrains.kover.test.functional.verification.subpackage.SubPartiallyCoveredFirst' is 52.631600, but expected minimum is 100
instructions missed percentage for class 'org.jetbrains.kover.test.functional.verification.subpackage.SubPartiallyCoveredSecond' is 66.216200, but expected minimum is 100
Rule 'fully covered instructions by packages' violated:
instructions covered percentage for package 'org.jetbrains.kover.test.functional.verification' is 48.275900, but expected minimum is 100
instructions covered percentage for package 'org.jetbrains.kover.test.functional.verification.subpackage' is 43.085100, but expected minimum is 100
Rule 'branches by classes' violated:
branches covered count for class 'org.jetbrains.kover.test.functional.verification.FullyCovered' is 0, but expected minimum is 1000
branches covered count for class 'org.jetbrains.kover.test.functional.verification.PartiallyCoveredFirst' is 2, but expected minimum is 1000
branches covered count for class 'org.jetbrains.kover.test.functional.verification.PartiallyCoveredSecond' is 1, but expected minimum is 1000
branches covered count for class 'org.jetbrains.kover.test.functional.verification.Uncovered' is 0, but expected minimum is 1000
branches covered count for class 'org.jetbrains.kover.test.functional.verification.subpackage.SubFullyCovered' is 0, but expected minimum is 1000
branches covered count for class 'org.jetbrains.kover.test.functional.verification.subpackage.SubPartiallyCoveredFirst' is 0, but expected minimum is 1000
branches covered count for class 'org.jetbrains.kover.test.functional.verification.subpackage.SubPartiallyCoveredSecond' is 1, but expected minimum is 1000
branches covered count for class 'org.jetbrains.kover.test.functional.verification.subpackage.SubUncovered' is 0, but expected minimum is 1000
Rule 'missed packages' violated:
lines missed count for package 'org.jetbrains.kover.test.functional.verification' is 23, but expected maximum is 1
lines missed count for package 'org.jetbrains.kover.test.functional.verification.subpackage' is 24, but expected maximum is 1
""")

assertJaCoCoResult("""Rule violated for bundle :: lines covered count is 41, but expected maximum is 3
Rule violated for bundle :: lines covered ratio is 0.46, but expected minimum is 0.58
Rule violated for class org.jetbrains.kover.test.functional.verification.FullyCovered: branches covered count is 0, but expected minimum is 1000
Rule violated for class org.jetbrains.kover.test.functional.verification.FullyCovered: instructions missed ratio is 0, but expected minimum is 1
Rule violated for class org.jetbrains.kover.test.functional.verification.PartiallyCoveredFirst: branches covered count is 2, but expected minimum is 1000
Rule violated for class org.jetbrains.kover.test.functional.verification.PartiallyCoveredFirst: instructions missed ratio is 0, but expected minimum is 1
Rule violated for class org.jetbrains.kover.test.functional.verification.PartiallyCoveredSecond: branches covered count is 1, but expected minimum is 1000
Rule violated for class org.jetbrains.kover.test.functional.verification.PartiallyCoveredSecond: instructions missed ratio is 0, but expected minimum is 1
Rule violated for class org.jetbrains.kover.test.functional.verification.Uncovered: branches covered count is 0, but expected minimum is 1000
Rule violated for class org.jetbrains.kover.test.functional.verification.subpackage.SubFullyCovered: branches covered count is 0, but expected minimum is 1000
Rule violated for class org.jetbrains.kover.test.functional.verification.subpackage.SubFullyCovered: instructions missed ratio is 0, but expected minimum is 1
Rule violated for class org.jetbrains.kover.test.functional.verification.subpackage.SubPartiallyCoveredFirst: branches covered count is 0, but expected minimum is 1000
Rule violated for class org.jetbrains.kover.test.functional.verification.subpackage.SubPartiallyCoveredFirst: instructions missed ratio is 0, but expected minimum is 1
Rule violated for class org.jetbrains.kover.test.functional.verification.subpackage.SubPartiallyCoveredSecond: branches covered count is 1, but expected minimum is 1000
Rule violated for class org.jetbrains.kover.test.functional.verification.subpackage.SubPartiallyCoveredSecond: instructions missed ratio is 0, but expected minimum is 1
Rule violated for class org.jetbrains.kover.test.functional.verification.subpackage.SubUncovered: branches covered count is 0, but expected minimum is 1000
Rule violated for package org.jetbrains.kover.test.functional.verification.subpackage: instructions covered ratio is 0, but expected minimum is 1
Rule violated for package org.jetbrains.kover.test.functional.verification.subpackage: lines missed count is 24, but expected maximum is 1
Rule violated for package org.jetbrains.kover.test.functional.verification: instructions covered ratio is 0, but expected minimum is 1
Rule violated for package org.jetbrains.kover.test.functional.verification: lines missed count is 23, but expected maximum is 1""")
}
}
}
Expand Down
Expand Up @@ -99,11 +99,13 @@ private class RunResultImpl(

override val defaultBinaryReport: String
get() {
// IntelliJ is a default Engine
val extension = if (slice?.engine == CoverageEngineVendor.JACOCO) "exec" else "ic"
val extension = if (engineVendor == CoverageEngineVendor.JACOCO) "exec" else "ic"
return binaryReportsDirectory() + "/" + defaultTestTask(slice?.type ?: ProjectType.KOTLIN_JVM) + "." + extension
}

// IntelliJ is a default Engine
val engineVendor: CoverageEngineVendor = slice?.engine ?: CoverageEngineVendor.INTELLIJ

private val buildScriptFile: File = buildFile()
private val buildScript: String by lazy { buildScriptFile.readText() }

Expand Down Expand Up @@ -132,10 +134,16 @@ private class RunResultImpl(
File(buildDir, name).checker()
}

override fun xml(filename: String, checker: XmlReport.() -> Unit) {
override fun xml(filename: String, checker: XmlReportChecker.() -> Unit) {
val xmlFile = File(buildDir, filename)
if (!xmlFile.exists()) throw IllegalStateException("XML file '$filename' not found")
XmlReportImpl(this, xmlFile).checker()
XmlReportCheckerImpl(this, xmlFile).checker()
}

override fun verification(checker: VerifyReportChecker.() -> Unit) {
val verificationResultFile = File(buildDir, "reports/kover/verification/errors.txt")
if (!verificationResultFile.exists()) throw IllegalStateException("Verification result file '$verificationResultFile' not found")
VerifyReportCheckerImpl(this, verificationResultFile.readText()).checker()
}

override fun outcome(taskName: String, checker: TaskOutcome.() -> Unit) {
Expand Down Expand Up @@ -186,8 +194,20 @@ private class CounterImpl(val context: RunResultImpl, val symbol: String, val ty
}
}

private class VerifyReportCheckerImpl(val context: RunResultImpl, val content: String): VerifyReportChecker {
override fun assertIntelliJResult(expected: String) {
if (context.engineVendor != CoverageEngineVendor.INTELLIJ) return
assertEquals(expected, content, "Unexpected verification result for IntelliJ Engine")
}

override fun assertJaCoCoResult(expected: String) {
if (context.engineVendor != CoverageEngineVendor.JACOCO) return
assertEquals(expected, content, "Unexpected verification result for JaCoCo Engine")
}

}

private class XmlReportImpl(val context: RunResultImpl, file: File) : XmlReport {
private class XmlReportCheckerImpl(val context: RunResultImpl, file: File) : XmlReportChecker {
private val document = DocumentBuilderFactory.newInstance()
// This option disables checking the dtd file for JaCoCo XML file
.also { it.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false) }
Expand Down
Expand Up @@ -129,7 +129,9 @@ internal interface RunResult {

fun file(name: String, checker: File.() -> Unit)

fun xml(filename: String, checker: XmlReport.() -> Unit)
fun xml(filename: String, checker: XmlReportChecker.() -> Unit)

fun verification(checker: VerifyReportChecker.() -> Unit)

fun outcome(taskName: String, checker: TaskOutcome.() -> Unit)

Expand All @@ -151,7 +153,12 @@ internal interface Counter {
fun assertFullyCovered()
}

internal interface XmlReport {
internal interface VerifyReportChecker {
fun assertIntelliJResult(expected: String)
fun assertJaCoCoResult(expected: String)
}

internal interface XmlReportChecker {
fun classCounter(className: String, type: String = "INSTRUCTION"): Counter
fun methodCounter(className: String, methodName: String, type: String = "INSTRUCTION"): Counter
}
@@ -0,0 +1,96 @@
package org.jetbrains.kover.test.functional.verification

class FullyCovered {
fun function0(i: Int): String {
val j = i + 2
println("function0")
return "result=$j"
}

fun function1(i: Int): String {
val j = i + 2
println("function0")
return "result=$j"
}

fun name(): String? {
return this::class.simpleName
}
}

class PartiallyCoveredFirst {
fun function0(i: Int): String {
val j = i + 2
if (i > 0) {
println("GTZ")
} else if (i == 0) {
println("EZ")
} else {
println("LEZ")
}
println("function0")
return "result=$j"
}

fun function1(i: Int): String {
val j = i + 2
println("function1")
return "result=$j"
}

fun name(): String? {
return this::class.simpleName
}
}

class PartiallyCoveredSecond {
fun function0(i: Int): String {
val j = i + 2
println("function0")
return "result=$j"
}

fun function1(i: Int): String {
val j = i + 2
println("function1")
if (i > 0) {
println("GTZ")
} else if (i == 0) {
println("EZ")
} else {
println("LEZ")
}
println("function1")
return "result=$j"
}

fun name(): String? {
return this::class.simpleName
}
}
class Uncovered {
fun function0(i: Int): String {
val j = i + 2
println("function0")
return "result=$j"
}

fun function1(i: Int): String {
val j = i + 2
println("function1")
if (i > 0) {
println("GTZ")
} else {
println("LEZ")
}
return "result=$j"
}

fun name(): String? {
return this::class.simpleName
}
}




0 comments on commit f7f0d1c

Please sign in to comment.