From 79feaa0eb8822ceecf48f498ea54316b1635d110 Mon Sep 17 00:00:00 2001 From: Roman Elizarov Date: Mon, 12 Oct 2020 19:03:46 +0300 Subject: [PATCH] Refactor mpp/native build, introduce "concurrent" source set, test launcher (#2074) New source sets: * "concurrent" source set is shared between "jvm" and "native" * "native" source set is subdivided into "nativeDarwin" (Apple) and "nativeOther" (Linux, etc) Native tests are launched in two variants: * A default "test" task runs tests with memory leak checker from "mainNoExit" entry point. * A special "backgroundTest" task runs tests in a background worker from "mainBackground" entry point. Other build improvement: * Modernize old-style IDEA-active hacks to kts helper. * Extract versions of JS test runner dependencies. * Remove redundant google repo reference from android tests. --- build.gradle | 4 +- gradle.properties | 11 +- gradle/compile-native-multiplatform.gradle | 40 +++--- gradle/targets.gradle | 28 ----- gradle/test-mocha-js.gradle | 4 +- kotlinx-coroutines-core/build.gradle | 118 +++++++++++++++--- .../native/src/WorkerMain.native.kt | 8 ++ .../native/test/WorkerTest.kt | 4 +- .../nativeDarwin/src/WorkerMain.kt | 13 ++ .../nativeDarwin/test/Launcher.kt | 28 +++++ .../nativeOther/src/WorkerMain.kt | 7 ++ .../nativeOther/test/Launcher.kt | 23 ++++ 12 files changed, 206 insertions(+), 82 deletions(-) delete mode 100644 gradle/targets.gradle create mode 100644 kotlinx-coroutines-core/native/src/WorkerMain.native.kt create mode 100644 kotlinx-coroutines-core/nativeDarwin/src/WorkerMain.kt create mode 100644 kotlinx-coroutines-core/nativeDarwin/test/Launcher.kt create mode 100644 kotlinx-coroutines-core/nativeOther/src/WorkerMain.kt create mode 100644 kotlinx-coroutines-core/nativeOther/test/Launcher.kt diff --git a/build.gradle b/build.gradle index 153714e90c..79c7f3553e 100644 --- a/build.gradle +++ b/build.gradle @@ -89,8 +89,8 @@ buildscript { import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformType -// Hierarchical project structures are not fully supported in 1.3.7x MPP, enable conditionally for 1.4.x -if (VersionNumber.parse(kotlin_version) > VersionNumber.parse("1.3.79")) { +// todo:KLUDGE: Hierarchical project structures are not fully supported in IDEA, enable only for a regular built +if (!Idea.active) { ext.set("kotlin.mpp.enableGranularSourceSetsMetadata", "true") } diff --git a/gradle.properties b/gradle.properties index 196a4a9ac0..821f546e8a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -36,10 +36,12 @@ kotlin.js.compiler=both gradle_node_version=1.2.0 node_version=8.9.3 npm_version=5.7.1 -mocha_version=4.1.0 +mocha_version=6.2.2 mocha_headless_chrome_version=1.8.2 -mocha_teamcity_reporter_version=2.2.2 -source_map_support_version=0.5.3 +mocha_teamcity_reporter_version=3.0.0 +source_map_support_version=0.5.16 +jsdom_version=15.2.1 +jsdom_global_version=3.0.2 # Settings kotlin.incremental.multiplatform=true @@ -56,7 +58,6 @@ org.gradle.jvmargs=-Xmx2g # https://github.com/gradle/gradle/issues/11412 systemProp.org.gradle.internal.publish.checksums.insecure=true -# This is commented out, and the property is set conditionally in build.gradle, because 1.3.71 doesn't work with it. -# Once this property is set by default in new versions or 1.3.71 is dropped, either uncomment or remove this line. +# todo:KLUDGE: This is commented out, and the property is set conditionally in build.gradle, because IDEA doesn't work with it. #kotlin.mpp.enableGranularSourceSetsMetadata=true kotlin.mpp.enableCompatibilityMetadataVariant=true diff --git a/gradle/compile-native-multiplatform.gradle b/gradle/compile-native-multiplatform.gradle index 378e4f5f98..4487446799 100644 --- a/gradle/compile-native-multiplatform.gradle +++ b/gradle/compile-native-multiplatform.gradle @@ -13,36 +13,24 @@ kotlin { } targets { - if (project.ext.ideaActive) { - fromPreset(project.ext.ideaPreset, 'native') - } else { - addTarget(presets.linuxX64) - addTarget(presets.iosArm64) - addTarget(presets.iosArm32) - addTarget(presets.iosX64) - addTarget(presets.macosX64) - addTarget(presets.mingwX64) - addTarget(presets.tvosArm64) - addTarget(presets.tvosX64) - addTarget(presets.watchosArm32) - addTarget(presets.watchosArm64) - addTarget(presets.watchosX86) - } + addTarget(presets.linuxX64) + addTarget(presets.iosArm64) + addTarget(presets.iosArm32) + addTarget(presets.iosX64) + addTarget(presets.macosX64) + addTarget(presets.mingwX64) + addTarget(presets.tvosArm64) + addTarget(presets.tvosX64) + addTarget(presets.watchosArm32) + addTarget(presets.watchosArm64) + addTarget(presets.watchosX86) } sourceSets { nativeMain { dependsOn commonMain } - // Empty source set is required in order to have native tests task - nativeTest {} + nativeTest { dependsOn commonTest } - if (!project.ext.ideaActive) { - configure(nativeMainSets) { - dependsOn nativeMain - } - - configure(nativeTestSets) { - dependsOn nativeTest - } - } + configure(nativeMainSets) { dependsOn nativeMain } + configure(nativeTestSets) { dependsOn nativeTest } } } diff --git a/gradle/targets.gradle b/gradle/targets.gradle deleted file mode 100644 index 08f3d989aa..0000000000 --- a/gradle/targets.gradle +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. - */ - -/* - * This is a hack to avoid creating unsupported native source sets when importing project into IDEA - */ -project.ext.ideaActive = System.getProperty('idea.active') == 'true' - -kotlin { - targets { - def manager = project.ext.hostManager - def linuxEnabled = manager.isEnabled(presets.linuxX64.konanTarget) - def macosEnabled = manager.isEnabled(presets.macosX64.konanTarget) - def winEnabled = manager.isEnabled(presets.mingwX64.konanTarget) - - project.ext.isLinuxHost = linuxEnabled - project.ext.isMacosHost = macosEnabled - project.ext.isWinHost = winEnabled - - if (project.ext.ideaActive) { - def ideaPreset = presets.linuxX64 - if (macosEnabled) ideaPreset = presets.macosX64 - if (winEnabled) ideaPreset = presets.mingwX64 - project.ext.ideaPreset = ideaPreset - } - } -} diff --git a/gradle/test-mocha-js.gradle b/gradle/test-mocha-js.gradle index 6676dc9268..7de79b9939 100644 --- a/gradle/test-mocha-js.gradle +++ b/gradle/test-mocha-js.gradle @@ -86,8 +86,8 @@ task testMochaChrome(type: NodeTask, dependsOn: prepareMochaChrome) { task installDependenciesMochaJsdom(type: NpmTask, dependsOn: [npmInstall]) { args = ['install', "mocha@$mocha_version", - 'jsdom@15.2.1', - 'jsdom-global@3.0.2', + "jsdom@$jsdom_version", + "jsdom-global@$jsdom_global_version", "source-map-support@$source_map_support_version", '--no-save'] if (project.hasProperty("teamcity")) args += ["mocha-teamcity-reporter@$mocha_teamcity_reporter_version"] diff --git a/kotlinx-coroutines-core/build.gradle b/kotlinx-coroutines-core/build.gradle index 55c44088f5..f98f6a529c 100644 --- a/kotlinx-coroutines-core/build.gradle +++ b/kotlinx-coroutines-core/build.gradle @@ -3,13 +3,60 @@ */ apply plugin: 'org.jetbrains.kotlin.multiplatform' -apply from: rootProject.file("gradle/targets.gradle") apply from: rootProject.file("gradle/compile-jvm-multiplatform.gradle") apply from: rootProject.file("gradle/compile-common.gradle") apply from: rootProject.file("gradle/compile-js-multiplatform.gradle") apply from: rootProject.file("gradle/compile-native-multiplatform.gradle") apply from: rootProject.file('gradle/publish-npm-js.gradle') +/* ========================================================================== + Configure source sets structure for kotlinx-coroutines-core: + + TARGETS SOURCE SETS + ------- ---------------------------------------------- + + js -----------------------------------------------------+ + | + V + jvm -------------------------------> concurrent ---> common + ^ + ios \ | + macos | ---> nativeDarwin ---> native --+ + tvos | ^ + watchos / | + | + linux \ ---> nativeOther -------+ + mingw / + + ========================================================================== */ + +project.ext.sourceSetSuffixes = ["Main", "Test"] + +void defineSourceSet(newName, dependsOn, includedInPred) { + for (suffix in project.ext.sourceSetSuffixes) { + def newSS = kotlin.sourceSets.maybeCreate(newName + suffix) + for (dep in dependsOn) { + newSS.dependsOn(kotlin.sourceSets[dep + suffix]) + } + for (curSS in kotlin.sourceSets) { + def curName = curSS.name + if (curName.endsWith(suffix)) { + def prefix = curName.substring(0, curName.length() - suffix.length()) + if (includedInPred(prefix)) curSS.dependsOn(newSS) + } + } + } +} + +static boolean isNativeDarwin(String name) { return ["ios", "macos", "tvos", "watchos"].any { name.startsWith(it) } } +static boolean isNativeOther(String name) { return ["linux", "mingw"].any { name.startsWith(it) } } + +defineSourceSet("concurrent", ["common"]) { it in ["jvm", "native"] } +defineSourceSet("nativeDarwin", ["native"]) { isNativeDarwin(it) } +defineSourceSet("nativeOther", ["native"]) { isNativeOther(it) } + +/* ========================================================================== */ + /* * All platform plugins and configuration magic happens here instead of build.gradle * because JMV-only projects depend on core, thus core should always be initialized before configuration. @@ -18,7 +65,7 @@ kotlin { configure(sourceSets) { def srcDir = name.endsWith('Main') ? 'src' : 'test' def platform = name[0..-5] - kotlin.srcDir "$platform/$srcDir" + kotlin.srcDirs = ["$platform/$srcDir"] if (name == "jvmMain") { resources.srcDirs = ["$platform/resources"] } else if (name == "jvmTest") { @@ -31,12 +78,18 @@ kotlin { } configure(targets) { - def targetName = it.name - compilations.all { compilation -> - def compileTask = tasks.getByName(compilation.compileKotlinTaskName) - // binary compatibility support - if (targetName.contains("jvm") && compilation.compilationName == "main") { - compileTask.kotlinOptions.freeCompilerArgs += ["-Xdump-declarations-to=${buildDir}/visibilities.json"] + // Configure additional binaries and test runs -- one for each OS + if (["macos", "linux", "mingw"].any { name.startsWith(it) }) { + binaries { + // Test for memory leaks using a special entry point that does not exit but returns from main + binaries.getTest("DEBUG").freeCompilerArgs += ["-e", "kotlinx.coroutines.mainNoExit"] + // Configure a separate test where code runs in background + test("background", [org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType.DEBUG]) { + freeCompilerArgs += ["-e", "kotlinx.coroutines.mainBackground"] + } + } + testRuns { + background { setExecutionSourceFrom(binaries.backgroundDebugTest) } } } } @@ -54,23 +107,52 @@ compileKotlinMetadata { } } +// :KLUDGE: Idea.active: This is needed to workaround resolve problems after importing this project to IDEA +def configureNativeSourceSetPreset(name, preset) { + def hostMainCompilation = project.kotlin.targetFromPreset(preset).compilations.main + // Look for platform libraries in "implementation" for default source set + def implementationConfiguration = configurations[hostMainCompilation.defaultSourceSet.implementationMetadataConfigurationName] + // Now find the libraries: Finds platform libs & stdlib, but platform declarations are still not resolved due to IDE bugs + def hostNativePlatformLibs = files( + provider { + implementationConfiguration.findAll { + it.path.endsWith(".klib") || it.absolutePath.contains("klib${File.separator}platform") || it.absolutePath.contains("stdlib") + } + } + ) + // Add all those dependencies + for (suffix in sourceSetSuffixes) { + configure(kotlin.sourceSets[name + suffix]) { + dependencies.add(implementationMetadataConfigurationName, hostNativePlatformLibs) + } + } +} + +// :KLUDGE: Idea.active: Configure platform libraries for native source sets when working in IDEA +if (Idea.active) { + def manager = project.ext.hostManager + def linuxPreset = kotlin.presets.linuxX64 + def macosPreset = kotlin.presets.macosX64 + // linux should be always available (cross-compilation capable) -- use it as default + assert manager.isEnabled(linuxPreset.konanTarget) + // use macOS libs for nativeDarwin if available + def macosAvailable = manager.isEnabled(macosPreset.konanTarget) + // configure source sets + configureNativeSourceSetPreset("native", linuxPreset) + configureNativeSourceSetPreset("nativeOther", linuxPreset) + configureNativeSourceSetPreset("nativeDarwin", macosAvailable ? macosPreset : linuxPreset) +} + kotlin.sourceSets { jvmMain.dependencies { compileOnly "com.google.android:annotations:4.1.1.4" } jvmTest.dependencies { - // This is a workaround for https://youtrack.jetbrains.com/issue/KT-39037 - def excludingCurrentProject = { dependency -> - def result = project.dependencies.create(dependency) - result.exclude(module: project.name) - return result - } - - api(excludingCurrentProject("org.jetbrains.kotlinx:lincheck:$lincheck_version")) + api "org.jetbrains.kotlinx:lincheck:$lincheck_version" api "org.jetbrains.kotlinx:kotlinx-knit-test:$knit_version" api "com.esotericsoftware:kryo:4.0.0" - implementation(excludingCurrentProject(project(":android-unit-tests"))) + implementation project(":android-unit-tests") } } @@ -97,7 +179,7 @@ jvmTest { enableAssertions = true systemProperty 'java.security.manager', 'kotlinx.coroutines.TestSecurityManager' // 'stress' is required to be able to run all subpackage tests like ":jvmTests --tests "*channels*" -Pstress=true" - if (!project.ext.ideaActive && rootProject.properties['stress'] == null) { + if (!Idea.active && rootProject.properties['stress'] == null) { exclude '**/*StressTest.*' } systemProperty 'kotlinx.coroutines.scheduler.keep.alive.sec', '100000' // any unpark problem hangs test diff --git a/kotlinx-coroutines-core/native/src/WorkerMain.native.kt b/kotlinx-coroutines-core/native/src/WorkerMain.native.kt new file mode 100644 index 0000000000..84cc9f42b9 --- /dev/null +++ b/kotlinx-coroutines-core/native/src/WorkerMain.native.kt @@ -0,0 +1,8 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +// It is used in the main sources of native-mt branch +internal expect inline fun workerMain(block: () -> Unit) diff --git a/kotlinx-coroutines-core/native/test/WorkerTest.kt b/kotlinx-coroutines-core/native/test/WorkerTest.kt index 84acedac94..d6b5fad182 100644 --- a/kotlinx-coroutines-core/native/test/WorkerTest.kt +++ b/kotlinx-coroutines-core/native/test/WorkerTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2016-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. */ package kotlinx.coroutines @@ -19,6 +19,7 @@ class WorkerTest : TestBase() { delay(1) } }.result + worker.requestTermination() } @Test @@ -31,5 +32,6 @@ class WorkerTest : TestBase() { }.join() } }.result + worker.requestTermination() } } diff --git a/kotlinx-coroutines-core/nativeDarwin/src/WorkerMain.kt b/kotlinx-coroutines-core/nativeDarwin/src/WorkerMain.kt new file mode 100644 index 0000000000..3445cb9897 --- /dev/null +++ b/kotlinx-coroutines-core/nativeDarwin/src/WorkerMain.kt @@ -0,0 +1,13 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlinx.cinterop.* + +internal actual inline fun workerMain(block: () -> Unit) { + autoreleasepool { + block() + } +} diff --git a/kotlinx-coroutines-core/nativeDarwin/test/Launcher.kt b/kotlinx-coroutines-core/nativeDarwin/test/Launcher.kt new file mode 100644 index 0000000000..78ed765967 --- /dev/null +++ b/kotlinx-coroutines-core/nativeDarwin/test/Launcher.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import platform.CoreFoundation.* +import kotlin.native.concurrent.* +import kotlin.native.internal.test.* +import kotlin.system.* + +// This is a separate entry point for tests in background +fun mainBackground(args: Array) { + val worker = Worker.start(name = "main-background") + worker.execute(TransferMode.SAFE, { args.freeze() }) { + val result = testLauncherEntryPoint(it) + exitProcess(result) + } + CFRunLoopRun() + error("CFRunLoopRun should never return") +} + +// This is a separate entry point for tests with leak checker +fun mainNoExit(args: Array) { + workerMain { // autoreleasepool to make sure interop objects are properly freed + testLauncherEntryPoint(args) + } +} \ No newline at end of file diff --git a/kotlinx-coroutines-core/nativeOther/src/WorkerMain.kt b/kotlinx-coroutines-core/nativeOther/src/WorkerMain.kt new file mode 100644 index 0000000000..cac0530e4e --- /dev/null +++ b/kotlinx-coroutines-core/nativeOther/src/WorkerMain.kt @@ -0,0 +1,7 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +internal actual inline fun workerMain(block: () -> Unit) = block() diff --git a/kotlinx-coroutines-core/nativeOther/test/Launcher.kt b/kotlinx-coroutines-core/nativeOther/test/Launcher.kt new file mode 100644 index 0000000000..feddd4c097 --- /dev/null +++ b/kotlinx-coroutines-core/nativeOther/test/Launcher.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license. + */ + +package kotlinx.coroutines + +import kotlin.native.concurrent.* +import kotlin.native.internal.test.* +import kotlin.system.* + +// This is a separate entry point for tests in background +fun mainBackground(args: Array) { + val worker = Worker.start(name = "main-background") + worker.execute(TransferMode.SAFE, { args.freeze() }) { + val result = testLauncherEntryPoint(it) + exitProcess(result) + }.result // block main thread +} + +// This is a separate entry point for tests with leak checker +fun mainNoExit(args: Array) { + testLauncherEntryPoint(args) +} \ No newline at end of file