diff --git a/build.gradle b/build.gradle index 87aac2a1e0..c679109ca3 100644 --- a/build.gradle +++ b/build.gradle @@ -8,8 +8,8 @@ apply from: rootProject.file("gradle/experimental.gradle") def rootModule = "kotlinx.coroutines" def coreModule = "kotlinx-coroutines-core" // Not applicable for Kotlin plugin -def sourceless = ['kotlinx.coroutines', 'site', 'kotlinx-coroutines-bom', 'publication-validator'] -def internal = ['kotlinx.coroutines', 'site', 'benchmarks', 'js-stub', 'stdlib-stubs', 'publication-validator'] +def sourceless = ['kotlinx.coroutines', 'site', 'kotlinx-coroutines-bom', 'integration-testing'] +def internal = ['kotlinx.coroutines', 'site', 'benchmarks', 'js-stub', 'stdlib-stubs', 'integration-testing'] // Not published def unpublished = internal + ['example-frontend-js', 'android-unit-tests'] diff --git a/gradle.properties b/gradle.properties index e9ebae8041..eaf530679c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,13 +14,15 @@ knit_version=0.1.3 html_version=0.6.8 lincheck_version=2.5.3 dokka_version=0.9.16-rdev-2-mpp-hacks -byte_buddy_version=1.9.3 +byte_buddy_version=1.10.7 reactor_vesion=3.2.5.RELEASE reactive_streams_version=1.0.2 rxjava2_version=2.2.8 javafx_version=11.0.2 javafx_plugin_version=0.0.8 binary_compatibility_validator_version=0.2.2 +blockhound_version=1.0.2.RELEASE +jna_version=5.5.0 # Android versions android_version=4.1.1.4 diff --git a/integration-testing/README.md b/integration-testing/README.md new file mode 100644 index 0000000000..4754081a45 --- /dev/null +++ b/integration-testing/README.md @@ -0,0 +1,14 @@ +# Integration tests + +This is a supplementary subproject of kotlinx.coroutines that provides +integration tests. + +The tests are the following: +* `NpmPublicationValidator` tests that version of NPM artifact is correct and that it has neither source nor package dependencies on atomicfu + In order for the test to work, one needs to run gradle with `-PdryRun=true`. + `-PdryRun` affects `npmPublish` so that it only provides a packed publication + and does not in fact attempt to send the build for publication. +* `MavenPublicationValidator` depends on the published artifacts and tests artifacts binary content and absence of atomicfu in the classpath +* `DebugAgentTest` checks that the coroutine debugger can be run as a Java agent. + +All the available tests can be run with `integration-testing:test`. diff --git a/integration-testing/build.gradle b/integration-testing/build.gradle new file mode 100644 index 0000000000..060eea414f --- /dev/null +++ b/integration-testing/build.gradle @@ -0,0 +1,77 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +apply from: rootProject.file("gradle/compile-jvm.gradle") + +repositories { + mavenLocal() + mavenCentral() +} + +sourceSets { + npmTest { + kotlin + compileClasspath += sourceSets.test.runtimeClasspath + runtimeClasspath += sourceSets.test.runtimeClasspath + } + mavenTest { + kotlin + compileClasspath += sourceSets.test.runtimeClasspath + runtimeClasspath += sourceSets.test.runtimeClasspath + } + debugAgentTest { + kotlin + compileClasspath += sourceSets.test.runtimeClasspath + runtimeClasspath += sourceSets.test.runtimeClasspath + } +} + +task npmTest(type: Test) { + def sourceSet = sourceSets.npmTest + environment "projectRoot", project.rootDir + environment "deployVersion", version + def dryRunNpm = project.properties['dryRun'] + def doRun = dryRunNpm == "true" // so that we don't accidentally publish anything, especially before the test + onlyIf { doRun } + if (doRun) { // `onlyIf` only affects execution of the task, not the dependency subtree + dependsOn(project(':').getTasksByName("publishNpm", true)) + } + testClassesDirs = sourceSet.output.classesDirs + classpath = sourceSet.runtimeClasspath +} + +task mavenTest(type: Test) { + def sourceSet = sourceSets.mavenTest + dependsOn(project(':').getTasksByName("publishToMavenLocal", true)) + dependsOn.remove(project(':').getTasksByName("dokka", true)) + testClassesDirs = sourceSet.output.classesDirs + classpath = sourceSet.runtimeClasspath +} + +task debugAgentTest(type: Test) { + def sourceSet = sourceSets.debugAgentTest + dependsOn(project(':kotlinx-coroutines-debug').shadowJar) + jvmArgs ('-javaagent:' + project(':kotlinx-coroutines-debug').shadowJar.outputs.files.getFiles()[0]) + testClassesDirs = sourceSet.output.classesDirs + classpath = sourceSet.runtimeClasspath +} + +dependencies { + testCompile "org.jetbrains.kotlin:kotlin-stdlib-jdk8" + testCompile 'junit:junit:4.12' + npmTestCompile 'org.apache.commons:commons-compress:1.18' + npmTestCompile 'com.google.code.gson:gson:2.8.5' + mavenTestRuntimeOnly project(':kotlinx-coroutines-core') + mavenTestRuntimeOnly project(':kotlinx-coroutines-android') + debugAgentTestCompile project(':kotlinx-coroutines-core') + debugAgentTestCompile project(':kotlinx-coroutines-debug') +} + +compileTestKotlin { + kotlinOptions.jvmTarget = "1.8" +} + +test { + dependsOn([npmTest, mavenTest, debugAgentTest]) +} diff --git a/integration-testing/src/debugAgentTest/kotlin/DebugAgentTest.kt b/integration-testing/src/debugAgentTest/kotlin/DebugAgentTest.kt new file mode 100644 index 0000000000..925fe077ea --- /dev/null +++ b/integration-testing/src/debugAgentTest/kotlin/DebugAgentTest.kt @@ -0,0 +1,20 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ +import org.junit.* +import kotlinx.coroutines.* +import kotlinx.coroutines.debug.* +import java.io.* + +class DebugAgentTest { + + @Test + fun agentDumpsCoroutines() = runBlocking { + val baos = ByteArrayOutputStream() + DebugProbes.dumpCoroutines(PrintStream(baos)) + // if the agent works, then dumps should contain something, + // at least the fact that this test is running. + Assert.assertTrue(baos.toString().contains("agentDumpsCoroutines")) + } + +} diff --git a/publication-validator/src/test/kotlin/kotlinx/coroutines/tools/MavenPublicationValidator.kt b/integration-testing/src/mavenTest/kotlin/MavenPublicationValidator.kt similarity index 99% rename from publication-validator/src/test/kotlin/kotlinx/coroutines/tools/MavenPublicationValidator.kt rename to integration-testing/src/mavenTest/kotlin/MavenPublicationValidator.kt index 53fd65d31f..5089c535ae 100644 --- a/publication-validator/src/test/kotlin/kotlinx/coroutines/tools/MavenPublicationValidator.kt +++ b/integration-testing/src/mavenTest/kotlin/MavenPublicationValidator.kt @@ -6,7 +6,6 @@ package kotlinx.coroutines.validator import org.junit.* import org.junit.Assert.assertTrue -import java.io.* import java.util.jar.* class MavenPublicationValidator { diff --git a/publication-validator/src/test/kotlin/kotlinx/coroutines/tools/NpmPublicationValidator.kt b/integration-testing/src/npmTest/kotlin/NpmPublicationValidator.kt similarity index 100% rename from publication-validator/src/test/kotlin/kotlinx/coroutines/tools/NpmPublicationValidator.kt rename to integration-testing/src/npmTest/kotlin/NpmPublicationValidator.kt diff --git a/kotlinx-coroutines-core/jvm/src/scheduling/CoroutineScheduler.kt b/kotlinx-coroutines-core/jvm/src/scheduling/CoroutineScheduler.kt index 815fa26941..62cf80f7f8 100644 --- a/kotlinx-coroutines-core/jvm/src/scheduling/CoroutineScheduler.kt +++ b/kotlinx-coroutines-core/jvm/src/scheduling/CoroutineScheduler.kt @@ -951,3 +951,20 @@ internal class CoroutineScheduler( TERMINATED } } + +/** + * Checks if the thread is part of a thread pool that supports coroutines. + * This function is needed for integration with BlockHound. + */ +@Suppress("UNUSED") +@JvmName("isSchedulerWorker") +internal fun isSchedulerWorker(thread: Thread) = thread is CoroutineScheduler.Worker + +/** + * Checks if the thread is running a CPU-bound task. + * This function is needed for integration with BlockHound. + */ +@Suppress("UNUSED") +@JvmName("mayNotBlock") +internal fun mayNotBlock(thread: Thread) = thread is CoroutineScheduler.Worker && + thread.state == CoroutineScheduler.WorkerState.CPU_ACQUIRED diff --git a/kotlinx-coroutines-debug/README.md b/kotlinx-coroutines-debug/README.md index 4128e1078b..772433ff5f 100644 --- a/kotlinx-coroutines-debug/README.md +++ b/kotlinx-coroutines-debug/README.md @@ -13,6 +13,11 @@ suspension stacktraces. Additionally, it is possible to process the list of such coroutines via [DebugProbes.dumpCoroutinesInfo] or dump isolated parts of coroutines hierarchy referenced by a [Job] or [CoroutineScope] instances using [DebugProbes.printJob] and [DebugProbes.printScope] respectively. +This module also provides an automatic [BlockHound](https://github.com/reactor/BlockHound) integration +that detects when a blocking operation was called in a coroutine context that prohibits it. In order to use it, +please follow the BlockHound [quick start guide]( +https://github.com/reactor/BlockHound/blob/1.0.2.RELEASE/docs/quick_start.md). + ### Using in your project Add `kotlinx-coroutines-debug` to your project test dependencies: diff --git a/kotlinx-coroutines-debug/api/kotlinx-coroutines-debug.api b/kotlinx-coroutines-debug/api/kotlinx-coroutines-debug.api index 6061f03899..749c94619e 100644 --- a/kotlinx-coroutines-debug/api/kotlinx-coroutines-debug.api +++ b/kotlinx-coroutines-debug/api/kotlinx-coroutines-debug.api @@ -8,6 +8,11 @@ public final class kotlinx/coroutines/debug/CoroutineInfo { public fun toString ()Ljava/lang/String; } +public final class kotlinx/coroutines/debug/CoroutinesBlockHoundIntegration : reactor/blockhound/integration/BlockHoundIntegration { + public fun ()V + public fun applyTo (Lreactor/blockhound/BlockHound$Builder;)V +} + public final class kotlinx/coroutines/debug/DebugProbes { public static final field INSTANCE Lkotlinx/coroutines/debug/DebugProbes; public final fun dumpCoroutines (Ljava/io/PrintStream;)V diff --git a/kotlinx-coroutines-debug/build.gradle b/kotlinx-coroutines-debug/build.gradle index 7fc2e22369..91c459efba 100644 --- a/kotlinx-coroutines-debug/build.gradle +++ b/kotlinx-coroutines-debug/build.gradle @@ -22,6 +22,10 @@ dependencies { compileOnly "junit:junit:$junit_version" shadowDeps "net.bytebuddy:byte-buddy:$byte_buddy_version" shadowDeps "net.bytebuddy:byte-buddy-agent:$byte_buddy_version" + compileOnly "io.projectreactor.tools:blockhound:$blockhound_version" + testCompile "io.projectreactor.tools:blockhound:$blockhound_version" + runtime "net.java.dev.jna:jna:$jna_version" + runtime "net.java.dev.jna:jna-platform:$jna_version" } jar { @@ -35,5 +39,5 @@ shadowJar { classifier null // Shadow only byte buddy, do not package kotlin stdlib configurations = [project.configurations.shadowDeps] - relocate 'net.bytebuddy', 'kotlinx.coroutines.repackaged.net.bytebuddy' + relocate('net.bytebuddy', 'kotlinx.coroutines.repackaged.net.bytebuddy') } diff --git a/kotlinx-coroutines-debug/resources/META-INF/services/reactor.blockhound.integration.BlockHoundIntegration b/kotlinx-coroutines-debug/resources/META-INF/services/reactor.blockhound.integration.BlockHoundIntegration new file mode 100644 index 0000000000..c2f1e9cf38 --- /dev/null +++ b/kotlinx-coroutines-debug/resources/META-INF/services/reactor.blockhound.integration.BlockHoundIntegration @@ -0,0 +1 @@ +kotlinx.coroutines.debug.CoroutinesBlockHoundIntegration \ No newline at end of file diff --git a/kotlinx-coroutines-debug/src/CoroutinesBlockHoundIntegration.kt b/kotlinx-coroutines-debug/src/CoroutinesBlockHoundIntegration.kt new file mode 100644 index 0000000000..f89d2be23f --- /dev/null +++ b/kotlinx-coroutines-debug/src/CoroutinesBlockHoundIntegration.kt @@ -0,0 +1,16 @@ +@file:Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER") +package kotlinx.coroutines.debug + +import reactor.blockhound.BlockHound +import kotlinx.coroutines.scheduling.* +import reactor.blockhound.integration.* + +@Suppress("UNUSED") +public class CoroutinesBlockHoundIntegration: BlockHoundIntegration { + + override fun applyTo(builder: BlockHound.Builder) { + builder.addDynamicThreadPredicate { isSchedulerWorker(it) } + builder.nonBlockingThreadPredicate { p -> p.or { mayNotBlock(it) } } + } + +} diff --git a/kotlinx-coroutines-debug/src/internal/DebugProbesImpl.kt b/kotlinx-coroutines-debug/src/internal/DebugProbesImpl.kt index 090d3e5d89..8b7d8e7998 100644 --- a/kotlinx-coroutines-debug/src/internal/DebugProbesImpl.kt +++ b/kotlinx-coroutines-debug/src/internal/DebugProbesImpl.kt @@ -57,7 +57,7 @@ internal object DebugProbesImpl { public fun install(): Unit = coroutineStateLock.write { if (++installations > 1) return - ByteBuddyAgent.install() + ByteBuddyAgent.install(ByteBuddyAgent.AttachmentProvider.ForEmulatedAttachment.INSTANCE) val cl = Class.forName("kotlin.coroutines.jvm.internal.DebugProbesKt") val cl2 = Class.forName("kotlinx.coroutines.debug.DebugProbesKt") diff --git a/kotlinx-coroutines-debug/test/BlockHoundTest.kt b/kotlinx-coroutines-debug/test/BlockHoundTest.kt new file mode 100644 index 0000000000..ff5c95cdb1 --- /dev/null +++ b/kotlinx-coroutines-debug/test/BlockHoundTest.kt @@ -0,0 +1,73 @@ +package kotlinx.coroutines.debug +import kotlinx.coroutines.* +import org.junit.* +import reactor.blockhound.* + +class BlockHoundTest : TestBase() { + + @Before + fun init() { + BlockHound.install() + } + + @Test(expected = BlockingOperationError::class) + fun shouldDetectBlockingInDefault() = runTest { + withContext(Dispatchers.Default) { + Thread.sleep(1) + } + } + + @Test + fun shouldNotDetectBlockingInIO() = runTest { + withContext(Dispatchers.IO) { + Thread.sleep(1) + } + } + + @Test + fun shouldNotDetectNonblocking() = runTest { + withContext(Dispatchers.Default) { + val a = 1 + val b = 2 + assert(a + b == 3) + } + } + + @Test + fun testReusingThreads() = runTest { + val n = 100 + repeat(n) { + async(Dispatchers.IO) { + Thread.sleep(1) + } + } + repeat(n) { + async(Dispatchers.Default) { + } + } + repeat(n) { + async(Dispatchers.IO) { + Thread.sleep(1) + } + } + } + + @Test(expected = BlockingOperationError::class) + fun testReusingThreadsFailure() = runTest { + val n = 100 + repeat(n) { + async(Dispatchers.IO) { + Thread.sleep(1) + } + } + async(Dispatchers.Default) { + Thread.sleep(1) + } + repeat(n) { + async(Dispatchers.IO) { + Thread.sleep(1) + } + } + } + +} diff --git a/kotlinx-coroutines-debug/test/CoroutinesDumpTest.kt b/kotlinx-coroutines-debug/test/CoroutinesDumpTest.kt index 91bd4f287d..8507721e30 100644 --- a/kotlinx-coroutines-debug/test/CoroutinesDumpTest.kt +++ b/kotlinx-coroutines-debug/test/CoroutinesDumpTest.kt @@ -39,7 +39,7 @@ class CoroutinesDumpTest : DebugTestBase() { @Test fun testRunningCoroutine() = runBlocking { - val deferred = async(Dispatchers.Default) { + val deferred = async(Dispatchers.IO) { activeMethod(shouldSuspend = false) assertTrue(true) } @@ -70,7 +70,7 @@ class CoroutinesDumpTest : DebugTestBase() { @Test fun testRunningCoroutineWithSuspensionPoint() = runBlocking { - val deferred = async(Dispatchers.Default) { + val deferred = async(Dispatchers.IO) { activeMethod(shouldSuspend = true) yield() // tail-call } @@ -100,7 +100,7 @@ class CoroutinesDumpTest : DebugTestBase() { @Test fun testCreationStackTrace() = runBlocking { - val deferred = async(Dispatchers.Default) { + val deferred = async(Dispatchers.IO) { activeMethod(shouldSuspend = true) } @@ -129,7 +129,7 @@ class CoroutinesDumpTest : DebugTestBase() { @Test fun testFinishedCoroutineRemoved() = runBlocking { - val deferred = async(Dispatchers.Default) { + val deferred = async(Dispatchers.IO) { activeMethod(shouldSuspend = true) } @@ -149,7 +149,10 @@ class CoroutinesDumpTest : DebugTestBase() { if (shouldSuspend) yield() notifyCoroutineStarted() while (coroutineContext[Job]!!.isActive) { - runCatching { Thread.sleep(60_000) } + try { + Thread.sleep(60_000) + } catch (_ : InterruptedException) { + } } } diff --git a/kotlinx-coroutines-debug/test/RunningThreadStackMergeTest.kt b/kotlinx-coroutines-debug/test/RunningThreadStackMergeTest.kt index c0b7f50134..85aa657be4 100644 --- a/kotlinx-coroutines-debug/test/RunningThreadStackMergeTest.kt +++ b/kotlinx-coroutines-debug/test/RunningThreadStackMergeTest.kt @@ -133,7 +133,7 @@ class RunningThreadStackMergeTest : DebugTestBase() { } private fun CoroutineScope.launchEscapingCoroutineWithoutContext() { - launch(Dispatchers.Default) { + launch(Dispatchers.IO) { suspendingFunctionWithoutContext() assertTrue(true) } diff --git a/kotlinx-coroutines-debug/test/StracktraceUtils.kt b/kotlinx-coroutines-debug/test/StracktraceUtils.kt index 12a39c0041..8c591ebd44 100644 --- a/kotlinx-coroutines-debug/test/StracktraceUtils.kt +++ b/kotlinx-coroutines-debug/test/StracktraceUtils.kt @@ -13,7 +13,7 @@ public fun String.trimStackTrace(): String = .replace(Regex("#[0-9]+"), "") .replace(Regex("(?<=\tat )[^\n]*/"), "") .replace(Regex("\t"), "") - .replace("sun.misc.Unsafe.park", "jdk.internal.misc.Unsafe.park") // JDK8->JDK11 + .replace("sun.misc.Unsafe.", "jdk.internal.misc.Unsafe.") // JDK8->JDK11 .applyBackspace() public fun String.applyBackspace(): String { @@ -62,6 +62,31 @@ public fun verifyDump(vararg traces: String, ignoredCoroutine: String? = null, f } } +/** Clean the stacktraces from artifacts of BlockHound instrumentation + * + * BlockHound works by switching a native call by a class generated with ByteBuddy, which, if the blocking + * call is allowed in this context, in turn calls the real native call that is now available under a + * different name. + * + * The traces thus undergo the following two changes when the execution is instrumented: + * - The original native call is replaced with a non-native one with the same FQN, and + * - An additional native call is placed on top of the stack, with the original name that also has + * `$$BlockHound$$_` prepended at the last component. + */ +private fun cleanBlockHoundTraces(frames: List): List { + var result = mutableListOf() + val blockHoundSubstr = "\$\$BlockHound\$\$_" + var i = 0 + while (i < frames.size) { + result.add(frames[i].replace(blockHoundSubstr, "")) + if (frames[i].contains(blockHoundSubstr)) { + i += 1 + } + i += 1 + } + return result +} + public fun verifyDump(vararg traces: String, ignoredCoroutine: String? = null) { val baos = ByteArrayOutputStream() DebugProbes.dumpCoroutines(PrintStream(baos)) @@ -85,7 +110,7 @@ public fun verifyDump(vararg traces: String, ignoredCoroutine: String? = null) { expected.withIndex().forEach { (index, trace) -> val actualTrace = actual[index].trimStackTrace().sanitizeAddresses() val expectedTrace = trace.trimStackTrace().sanitizeAddresses() - val actualLines = actualTrace.split("\n") + val actualLines = cleanBlockHoundTraces(actualTrace.split("\n")) val expectedLines = expectedTrace.split("\n") for (i in expectedLines.indices) { assertEquals(expectedLines[i], actualLines[i]) diff --git a/publication-validator/README.md b/publication-validator/README.md deleted file mode 100644 index a60ff00e3c..0000000000 --- a/publication-validator/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Publication validator - -This is a supplementary subproject of kotlinx.coroutines that provides a new -task, `testPublishing`, to test its publication correctness. - -The tests are the following: -* `NpmPublicationValidator` tests that version of NPM artifact is correct and that it has neither source nor package dependencies on atomicfu -* `MavenPublicationValidator` depends on the published artifacts and tests artifacts binary content and absence of atomicfu in the classpath - -To test publication, one needs to run gradle with `-PdryRun=true`, and the -task that actually does the testing is `publication-validator:test`. -`-PdryRun` affects `npmPublish` so that it only provides a packed publication -and does not in fact attempt to send the build for publication. diff --git a/publication-validator/build.gradle b/publication-validator/build.gradle deleted file mode 100644 index a22ccf46d2..0000000000 --- a/publication-validator/build.gradle +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -apply from: rootProject.file("gradle/compile-jvm.gradle") - -repositories { - mavenLocal() - mavenCentral() -} - -dependencies { - testCompile "org.jetbrains.kotlin:kotlin-stdlib-jdk8" - testCompile 'junit:junit:4.12' - testCompile 'org.apache.commons:commons-compress:1.18' - testCompile 'com.google.code.gson:gson:2.8.5' - testCompile project(':kotlinx-coroutines-core') - testCompile project(':kotlinx-coroutines-android') -} - -compileTestKotlin { - kotlinOptions.jvmTarget = "1.8" -} - -def dryRunNpm = properties['dryRun'] - -test { - onlyIf { dryRunNpm == "true" } // so that we don't accidentally publish anything, especially before the test - doFirst { println "Verifying publishing version $version" } // all modules share the same version - environment "projectRoot", project.rootDir - environment "deployVersion", version - if (dryRunNpm == "true") { // `onlyIf` only affects execution of the task, not the dependency subtree - dependsOn(project(':').getTasksByName("publishNpm", true) + - project(':').getTasksByName("publishToMavenLocal", true)) - dependsOn.remove(project(':').getTasksByName("dokka", true)) - } -} diff --git a/settings.gradle b/settings.gradle index 64ae2ffad3..95fcd7cb2d 100644 --- a/settings.gradle +++ b/settings.gradle @@ -45,4 +45,4 @@ if (!build_snapshot_train) { include('site') } -module('publication-validator') +module('integration-testing')