Make DebugProbes ready to production (#1862)

* Speed-up installed debug probes by splitting global probes lock to RW-lock, guard all state transitions with read lock and all read operations with write lock to guarantee a consistent snapshot
* Prevent IllegalStateException during 'kill -5' command
* Introduce flag to disable creation stacktrace capturing in DebugProbes
* Support proposed changes in JUnit4 rules

Fixes #1379
Fixes #1372

README.md

@@ -277,6 +277,7 @@ The `develop` branch is pushed to `master` during release.
 [ListenableFuture.await]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-guava/kotlinx.coroutines.guava/com.google.common.util.concurrent.-listenable-future/await.html
 <!--- MODULE kotlinx-coroutines-play-services -->
 <!--- INDEX kotlinx.coroutines.tasks -->
+[Task.await]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-play-services/kotlinx.coroutines.tasks/com.google.android.gms.tasks.-task/await.html
 <!--- MODULE kotlinx-coroutines-reactive -->
 <!--- INDEX kotlinx.coroutines.reactive -->
 [Publisher.collect]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-reactive/kotlinx.coroutines.reactive/org.reactivestreams.-publisher/collect.html

kotlinx-coroutines-debug/README.md

@@ -55,9 +55,20 @@ stacktraces will be dumped to the console.
 
 ### Using as JVM agent
 
-It is possible to use this module as a standalone JVM agent to enable debug probes on the application startup.
+Debug module can also be used as a standalone JVM agent to enable debug probes on the application startup.
 You can run your application with an additional argument: `-javaagent:kotlinx-coroutines-debug-1.3.4.jar`.
 Additionally, on Linux and Mac OS X you can use `kill -5 $pid` command in order to force your application to print all alive coroutines.
+When used as Java agent, `"kotlinx.coroutines.debug.enable.creation.stack.trace"` system property can be used to control 
+[DebugProbes.enableCreationStackTraces] along with agent startup.
+
+### Using in production environment
+
+It is possible to run an application in production environments with debug probes in order to monitor its 
+state and improve its observability. 
+For that, it is strongly recommended to switch off [DebugProbes.enableCreationStackTraces] property to significantly 
+reduce the overhead of debug probes and make it insignificant.
+With creation stack-traces disabled, the typical overhead of enabled debug probes is a single-digit percentage of the total
+application throughput.
 
 
 ### Example of usage
@@ -128,8 +139,13 @@ Dumping only deferred
 
 ### Status of the API
 
-API is purely experimental and it is not guaranteed that it won't be changed (while it is marked as `@ExperimentalCoroutinesApi`).
-Do not use this module in production environment and do not rely on the format of the data produced by [DebugProbes]. 
+API is experimental, and it is not guaranteed it won't be changed (while it is marked as `@ExperimentalCoroutinesApi`).
+Like the rest of experimental API, `DebugProbes` is carefully designed, tested and ready to use in both test and production 
+environments. It is marked as experimental to leave us the room to enrich the output data in a potentially backwards incompatible manner
+to further improve diagnostics and debugging experience.
+
+The output format of [DebugProbes] can be changed in the future and it is not recommended to rely on the string representation
+of the dump programmatically.
 
 ### Debug agent and Android
 
@@ -161,6 +177,7 @@ java.lang.NoClassDefFoundError: Failed resolution of: Ljava/lang/management/Mana
 [DebugProbes.dumpCoroutinesInfo]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-debug/kotlinx.coroutines.debug/-debug-probes/dump-coroutines-info.html
 [DebugProbes.printJob]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-debug/kotlinx.coroutines.debug/-debug-probes/print-job.html
 [DebugProbes.printScope]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-debug/kotlinx.coroutines.debug/-debug-probes/print-scope.html
+[DebugProbes.enableCreationStackTraces]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-debug/kotlinx.coroutines.debug/-debug-probes/enable-creation-stack-traces.html
 <!--- INDEX kotlinx.coroutines.debug.junit4 -->
 [CoroutinesTimeout]: https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-debug/kotlinx.coroutines.debug.junit4/-coroutines-timeout/index.html
 <!--- END -->

kotlinx-coroutines-debug/api/kotlinx-coroutines-debug.api

@@ -13,6 +13,7 @@ public final class kotlinx/coroutines/debug/DebugProbes {
 	public final fun dumpCoroutines (Ljava/io/PrintStream;)V
 	public static synthetic fun dumpCoroutines$default (Lkotlinx/coroutines/debug/DebugProbes;Ljava/io/PrintStream;ILjava/lang/Object;)V
 	public final fun dumpCoroutinesInfo ()Ljava/util/List;
+	public final fun getEnableCreationStackTraces ()Z
 	public final fun getSanitizeStackTraces ()Z
 	public final fun install ()V
 	public final fun isInstalled ()Z
@@ -22,6 +23,7 @@ public final class kotlinx/coroutines/debug/DebugProbes {
 	public final fun printScope (Lkotlinx/coroutines/CoroutineScope;Ljava/io/PrintStream;)V
 	public static synthetic fun printScope$default (Lkotlinx/coroutines/debug/DebugProbes;Lkotlinx/coroutines/CoroutineScope;Ljava/io/PrintStream;ILjava/lang/Object;)V
 	public final fun scopeToString (Lkotlinx/coroutines/CoroutineScope;)Ljava/lang/String;
+	public final fun setEnableCreationStackTraces (Z)V
 	public final fun setSanitizeStackTraces (Z)V
 	public final fun uninstall ()V
 	public final fun withDebugProbes (Lkotlin/jvm/functions/Function0;)V
@@ -35,17 +37,27 @@ public final class kotlinx/coroutines/debug/State : java/lang/Enum {
 	public static fun values ()[Lkotlinx/coroutines/debug/State;
 }
 
+public synthetic class kotlinx/coroutines/debug/internal/DebugProbesImplSequenceNumberRefVolatile {
+	public fun <init> (J)V
+}
+
 public final class kotlinx/coroutines/debug/junit4/CoroutinesTimeout : org/junit/rules/TestRule {
 	public static final field Companion Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout$Companion;
 	public fun <init> (JZ)V
 	public synthetic fun <init> (JZILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public fun <init> (JZZ)V
+	public synthetic fun <init> (JZZILkotlin/jvm/internal/DefaultConstructorMarker;)V
 	public fun apply (Lorg/junit/runners/model/Statement;Lorg/junit/runner/Description;)Lorg/junit/runners/model/Statement;
 }
 
 public final class kotlinx/coroutines/debug/junit4/CoroutinesTimeout$Companion {
+	public final fun seconds (I)Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout;
 	public final fun seconds (IZ)Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout;
+	public final fun seconds (IZZ)Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout;
+	public final fun seconds (J)Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout;
 	public final fun seconds (JZ)Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout;
-	public static synthetic fun seconds$default (Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout$Companion;IZILjava/lang/Object;)Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout;
-	public static synthetic fun seconds$default (Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout$Companion;JZILjava/lang/Object;)Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout;
+	public final fun seconds (JZZ)Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout;
+	public static synthetic fun seconds$default (Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout$Companion;IZZILjava/lang/Object;)Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout;
+	public static synthetic fun seconds$default (Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout$Companion;JZZILjava/lang/Object;)Lkotlinx/coroutines/debug/junit4/CoroutinesTimeout;
 }
 

kotlinx-coroutines-debug/src/AgentPremain.kt

@@ -11,17 +11,28 @@ import java.lang.instrument.*
 @Suppress("unused")
 internal object AgentPremain {
 
+    private val enableCreationStackTraces =
+        System.getProperty("kotlinx.coroutines.debug.enable.creation.stack.trace")?.toBoolean()
+            ?: DebugProbes.enableCreationStackTraces
+
     @JvmStatic
     public fun premain(args: String?, instrumentation: Instrumentation) {
         Installer.premain(args, instrumentation)
+        DebugProbes.enableCreationStackTraces = enableCreationStackTraces
         DebugProbes.install()
         installSignalHandler()
     }
 
     private fun installSignalHandler() {
         try {
             Signal.handle(Signal("TRAP")) { // kill -5
-                DebugProbes.dumpCoroutines()
+                if (DebugProbes.isInstalled) {
+                    // Case with 'isInstalled' changed between this check-and-act is not considered
+                    // a real debug probes use-case, thus is not guarded against.
+                    DebugProbes.dumpCoroutines()
+                } else {
+                    println("""Cannot perform coroutines dump, debug probes are disabled""")
+                }
             }
         } catch (t: Throwable) {
             System.err.println("Failed to install signal handler: $t")

kotlinx-coroutines-debug/src/CoroutineInfo.kt

@@ -16,7 +16,7 @@ import kotlin.coroutines.jvm.internal.*
 @ExperimentalCoroutinesApi
 public class CoroutineInfo internal constructor(
     val context: CoroutineContext,
-    private val creationStackBottom: CoroutineStackFrame,
+    private val creationStackBottom: CoroutineStackFrame?,
     @JvmField internal val sequenceNumber: Long
 ) {
 
@@ -28,6 +28,7 @@ public class CoroutineInfo internal constructor(
 
     /**
      * Creation stacktrace of the coroutine.
+     * Can be empty if [DebugProbes.enableCreationStackTraces] is not set.
      */
     public val creationStackTrace: List<StackTraceElement> get() = creationStackTrace()
 
@@ -66,8 +67,9 @@ public class CoroutineInfo internal constructor(
     }
 
     private fun creationStackTrace(): List<StackTraceElement> {
+        val bottom = creationStackBottom ?: return emptyList()
         // Skip "Coroutine creation stacktrace" frame
-        return sequence<StackTraceElement> { yieldFrames(creationStackBottom.callerFrame) }.toList()
+        return sequence<StackTraceElement> { yieldFrames(bottom.callerFrame) }.toList()
     }
 
     private tailrec suspend fun SequenceScope<StackTraceElement>.yieldFrames(frame: CoroutineStackFrame?) {

kotlinx-coroutines-debug/src/DebugProbes.kt

@@ -28,20 +28,30 @@ import kotlin.coroutines.*
  * * `probeCoroutineCreated` is invoked on every coroutine creation using stdlib intrinsics.
  *
  * Overhead:
- *  * Every created coroutine is stored in a weak hash map, thus adding additional GC pressure.
- *  * On every created coroutine, stacktrace of the current thread is dumped.
- *  * On every `resume` and `suspend`, [WeakHashMap] is updated under a global lock.
+ *  * Every created coroutine is stored in a concurrent hash map and hash map is looked up and
+ *    updated on each suspension and resumption.
+ *  * If [DebugProbes.enableCreationStackTraces] is enabled, stack trace of the current thread is captured on
+ *    each created coroutine that is a rough equivalent of throwing an exception per each created coroutine.
  */
 @ExperimentalCoroutinesApi
 public object DebugProbes {
 
     /**
-     * Whether coroutine creation stacktraces should be sanitized.
+     * Whether coroutine creation stack traces should be sanitized.
      * Sanitization removes all frames from `kotlinx.coroutines` package except
      * the first one and the last one to simplify diagnostic.
      */
     public var sanitizeStackTraces: Boolean = true
 
+    /**
+     * Whether coroutine creation stack traces should be captured.
+     * When enabled, for each created coroutine a stack trace of the current
+     * thread is captured and attached to the coroutine.
+     * This option can be useful during local debug sessions, but is recommended
+     * to be disabled in production environments to avoid stack trace dumping overhead.
+     */
+    public var enableCreationStackTraces: Boolean = true
+
     /**
      * Determines whether debug probes were [installed][DebugProbes.install].
      */
@@ -132,5 +142,5 @@ public object DebugProbes {
 internal fun probeCoroutineResumed(frame: Continuation<*>) = DebugProbesImpl.probeCoroutineResumed(frame)
 
 internal fun probeCoroutineSuspended(frame: Continuation<*>) = DebugProbesImpl.probeCoroutineSuspended(frame)
-internal fun <T> probeCoroutineCreated(completion: kotlin.coroutines.Continuation<T>): kotlin.coroutines.Continuation<T> =
+internal fun <T> probeCoroutineCreated(completion: Continuation<T>): Continuation<T> =
     DebugProbesImpl.probeCoroutineCreated(completion)