From bd68eb2651be90c23b51be406878776d02088627 Mon Sep 17 00:00:00 2001 From: "nima.taheri@hootsuite.com" Date: Sun, 21 Aug 2022 14:22:34 -0700 Subject: [PATCH] feat: set sorting reporter as complete if test run failed If test runner is sorted, it buffers all events to be able to print them sorted. If test runner fails unexpectedly it does not flush buffered events until an internal timer kicks off; that won't work well with SBT where all loggers are discarded by the time the timer is triggered. --- .../scala/org/scalatest/tools/Framework.scala | 263 +++++++++--------- .../org/scalatest/tools/FrameworkSuite.scala | 241 ++++++++-------- .../tools/scalasbt/BadBeforeAsyncSuite.scala | 31 +++ 3 files changed, 294 insertions(+), 241 deletions(-) create mode 100644 jvm/scalatest-test/src/test/scala/org/scalatest/tools/scalasbt/BadBeforeAsyncSuite.scala diff --git a/jvm/core/src/main/scala/org/scalatest/tools/Framework.scala b/jvm/core/src/main/scala/org/scalatest/tools/Framework.scala index c638e7166a..35b26c5ab4 100644 --- a/jvm/core/src/main/scala/org/scalatest/tools/Framework.scala +++ b/jvm/core/src/main/scala/org/scalatest/tools/Framework.scala @@ -206,7 +206,7 @@ class Framework extends SbtFramework { * @return ScalaTest */ def name = "ScalaTest" - + private val resultHolder = new SuiteResultHolder() /** @@ -222,12 +222,12 @@ class Framework extends SbtFramework { def superclassName = "org.scalatest.Suite" def isModule = false def requireNoArgConstructor = true - }, + }, new AnnotatedFingerprint { def annotationName = "org.scalatest.WrapWith" def isModule = false }) - + private def runSuite( taskDefinition: TaskDef, rerunSuiteId: String, @@ -235,11 +235,11 @@ class Framework extends SbtFramework { loader: ClassLoader, suiteSortingReporter: SuiteSortingReporter, tracker: Tracker, - eventHandler: EventHandler, + eventHandler: EventHandler, tagsToInclude: Set[String], tagsToExclude: Set[String], selectors: Array[Selector], - explicitlySpecified: Boolean, + explicitlySpecified: Boolean, configMap: ConfigMap, summaryCounter: SummaryCounter, statefulStatus: Option[ScalaTestStatefulStatus], @@ -249,7 +249,7 @@ class Framework extends SbtFramework { presentAllDurations: Boolean, presentInColor: Boolean, presentShortStackTraces: Boolean, - presentFullStackTraces: Boolean, + presentFullStackTraces: Boolean, presentUnformatted: Boolean, presentReminder: Boolean, presentReminderWithShortStackTraces: Boolean, @@ -262,8 +262,8 @@ class Framework extends SbtFramework { val report = new SbtReporter(rerunSuiteId, taskDefinition.fullyQualifiedName, taskDefinition.fingerprint, eventHandler, suiteSortingReporter, summaryCounter) val formatter = formatterForSuiteStarting(suite) val suiteClassName = getSuiteClassName(suite) - - val filter = + + val filter = if ((selectors.length == 1 && selectors(0).isInstanceOf[SuiteSelector] && !explicitlySpecified)) // selectors will always at least have one SuiteSelector, according to javadoc of TaskDef Filter(if (tagsToInclude.isEmpty) None else Some(tagsToInclude), tagsToExclude) else { @@ -274,10 +274,10 @@ class Framework extends SbtFramework { selectors.foreach { selector => selector match { - case suiteSelector: SuiteSelector => + case suiteSelector: SuiteSelector => suiteTags = mergeMap[String, Set[String]](List(suiteTags, Map(suite.suiteId -> Set(SELECTED_TAG)))) { _ ++ _ } case testSelector: TestSelector => - testTags = mergeMap[String, Map[String, Set[String]]](List(testTags, Map(suite.suiteId -> Map(testSelector.testName -> Set(SELECTED_TAG))))) { (testMap1, testMap2) => + testTags = mergeMap[String, Map[String, Set[String]]](List(testTags, Map(suite.suiteId -> Map(testSelector.testName -> Set(SELECTED_TAG))))) { (testMap1, testMap2) => mergeMap[String, Set[String]](List(testMap1, testMap2)) { _ ++ _} } hasTest = true @@ -288,11 +288,11 @@ class Framework extends SbtFramework { mergeMap[String, Set[String]](List(testMap1, testMap2)) { _ ++ _} } hasTest = true - case nestedSuiteSelector: NestedSuiteSelector => + case nestedSuiteSelector: NestedSuiteSelector => suiteTags = mergeMap[String, Set[String]](List(suiteTags, Map(nestedSuiteSelector.suiteId -> Set(SELECTED_TAG)))) { _ ++ _ } hasNested = true - case nestedTestSelector: NestedTestSelector => - testTags = mergeMap[String, Map[String, Set[String]]](List(testTags, Map(nestedTestSelector.suiteId -> Map(nestedTestSelector.testName -> Set(SELECTED_TAG))))) { (testMap1, testMap2) => + case nestedTestSelector: NestedTestSelector => + testTags = mergeMap[String, Map[String, Set[String]]](List(testTags, Map(nestedTestSelector.suiteId -> Map(nestedTestSelector.testName -> Set(SELECTED_TAG))))) { (testMap1, testMap2) => mergeMap[String, Set[String]](List(testMap1, testMap2)) { _ ++ _} } hasNested = true @@ -337,8 +337,9 @@ class Framework extends SbtFramework { } } } - catch { + catch { case e: Throwable => { + suiteSortingReporter.completedTests(suite.suiteId) // TODO: Could not get this from Resources. Got: // java.util.MissingResourceException: Can't find bundle for base name org.scalatest.ScalaTestBundle, locale en_US @@ -370,25 +371,25 @@ class Framework extends SbtFramework { case None => // Do nothing } } - + Array.empty } - + private class ScalaTestTask( - taskDefinition: TaskDef, + taskDefinition: TaskDef, loader: ClassLoader, suiteSortingReporter: SuiteSortingReporter, tracker: Tracker, - tagsToInclude: Set[String], + tagsToInclude: Set[String], tagsToExclude: Set[String], selectors: Array[Selector], - explicitlySpecified: Boolean, - configMap: ConfigMap, + explicitlySpecified: Boolean, + configMap: ConfigMap, summaryCounter: SummaryCounter, statusList: LinkedBlockingQueue[Status], useSbtLogInfoReporter: Boolean, presentAllDurations: Boolean, - presentInColor: Boolean, + presentInColor: Boolean, presentShortStackTraces: Boolean, presentFullStackTraces: Boolean, presentUnformatted: Boolean, @@ -401,30 +402,30 @@ class Framework extends SbtFramework { configSet: Set[ReporterConfigParam], execService: ExecutorService ) extends Task { - + def loadSuiteClass = { try { Class.forName(taskDefinition.fullyQualifiedName, true, loader) } catch { - case e: Exception => + case e: Exception => throw new IllegalArgumentException("Unable to load class: " + taskDefinition.fullyQualifiedName) } } - + lazy val suiteClass = loadSuiteClass lazy val accessible = isAccessibleSuite(suiteClass) lazy val runnable = isRunnable(suiteClass) - lazy val shouldDiscover = + lazy val shouldDiscover = taskDefinition.explicitlySpecified || ((accessible || runnable) && isDiscoverableSuite(suiteClass)) - - def tags = - for { + + def tags = + for { a <- suiteClass.getAnnotations annotationClass = a.annotationType - if (annotationClass.isAnnotationPresent(classOf[TagAnnotation]) || annotationClass.isAssignableFrom(classOf[TagAnnotation])) + if (annotationClass.isAnnotationPresent(classOf[TagAnnotation]) || annotationClass.isAssignableFrom(classOf[TagAnnotation])) } yield { - val value = + val value = if (a.isInstanceOf[TagAnnotation]) a.asInstanceOf[TagAnnotation].value else @@ -434,7 +435,7 @@ class Framework extends SbtFramework { else value } - + def execute(eventHandler: EventHandler, loggers: Array[Logger]) = { if (accessible || runnable) { val suite = @@ -450,7 +451,7 @@ class Framework extends SbtFramework { constructor.get.newInstance(suiteClass).asInstanceOf[Suite] } else - suiteClass.newInstance.asInstanceOf[Suite] + suiteClass.newInstance.asInstanceOf[Suite] } catch { case t: Throwable => new DeferredAbortedSuite(suiteClass.getName, suiteClass.getName, t) } @@ -497,7 +498,7 @@ class Framework extends SbtFramework { tagsToInclude, tagsToExclude, selectors, - explicitlySpecified, + explicitlySpecified, configMap, summaryCounter, None, @@ -516,54 +517,54 @@ class Framework extends SbtFramework { execService ) } - else + else throw new IllegalArgumentException("Class " + taskDefinition.fullyQualifiedName + " is neither accessible accesible org.scalatest.Suite nor runnable.") } - + def taskDef = taskDefinition } - + private[tools] class SummaryCounter { val testsSucceededCount, testsFailedCount, testsIgnoredCount, testsPendingCount, testsCanceledCount, suitesCompletedCount, suitesAbortedCount, scopesPendingCount = new AtomicInteger val reminderEventsQueue = new LinkedBlockingQueue[ExceptionalEvent] - - def incrementTestsSucceededCount(): Unit = { - testsSucceededCount.incrementAndGet() + + def incrementTestsSucceededCount(): Unit = { + testsSucceededCount.incrementAndGet() } - + def incrementTestsFailedCount(): Unit = { testsFailedCount.incrementAndGet() } - + def incrementTestsIgnoredCount(): Unit = { testsIgnoredCount.incrementAndGet() } - + def incrementTestsPendingCount(): Unit = { testsPendingCount.incrementAndGet() } - + def incrementTestsCanceledCount(): Unit = { testsCanceledCount.incrementAndGet() } - + def incrementSuitesCompletedCount(): Unit = { suitesCompletedCount.incrementAndGet() } - + def incrementSuitesAbortedCount(): Unit = { suitesAbortedCount.incrementAndGet() } - + def incrementScopesPendingCount(): Unit = { scopesPendingCount.incrementAndGet() } - + def recordReminderEvents(events: ExceptionalEvent): Unit = { reminderEventsQueue.put(events) } } - + private class SbtLogInfoReporter( loggers: Array[Logger], presentAllDurations: Boolean, @@ -603,7 +604,7 @@ class Framework extends SbtFramework { logger.info(text) } } - + override def apply(event: Event): Unit = { event match { case ee: ExceptionalEvent if presentReminder => @@ -633,16 +634,16 @@ class Framework extends SbtFramework { def dispose(): Unit = () } - + private[scalatest] class ScalaTestRunner( runArgs: Array[String], loader: ClassLoader, tagsToInclude: Set[String], tagsToExclude: Set[String], - membersOnly: List[String], + membersOnly: List[String], wildcard: List[String], autoSelectors: List[Selector], - configMap: ConfigMap, + configMap: ConfigMap, val repConfig: ReporterConfigurations, val useSbtLogInfoReporter: Boolean, val presentAllDurations: Boolean, @@ -669,7 +670,7 @@ class Framework extends SbtFramework { val tracker = new Tracker val summaryCounter = new SummaryCounter val runStartTime = System.currentTimeMillis - + val dispatchReporter = ReporterFactory.getDispatchReporter(repConfig, None, None, loader, Some(resultHolder), detectSlowpokes, slowpokeDetectionDelay, slowpokeDetectionPeriod) val suiteSortingReporter = @@ -680,7 +681,7 @@ class Framework extends SbtFramework { if (detectSlowpokes) dispatchReporter.registerSlowpokeReporter(suiteSortingReporter) - + dispatchReporter(RunStarting(tracker.nextOrdinal(), 0, configMap)) private val atomicThreadCounter = new AtomicInteger @@ -706,17 +707,17 @@ class Framework extends SbtFramework { Executors.newFixedThreadPool(poolSize, threadFactory) else Executors.newCachedThreadPool(threadFactory) - - private def createTask(td: TaskDef): ScalaTestTask = + + private def createTask(td: TaskDef): ScalaTestTask = new ScalaTestTask( - td, + td, loader, suiteSortingReporter, tracker, tagsToInclude, tagsToExclude, td.selectors ++ autoSelectors, - td.explicitlySpecified, + td.explicitlySpecified, configMap, summaryCounter, statusList, @@ -735,22 +736,22 @@ class Framework extends SbtFramework { configSet, execSvc ) - - private def filterWildcard(paths: List[String], taskDefs: Array[TaskDef]): Array[TaskDef] = + + private def filterWildcard(paths: List[String], taskDefs: Array[TaskDef]): Array[TaskDef] = taskDefs.filter(td => paths.exists(td.fullyQualifiedName.startsWith(_))) - + private def filterMembersOnly(paths: List[String], taskDefs: Array[TaskDef]): Array[TaskDef] = taskDefs.filter { td => paths.exists(path => td.fullyQualifiedName.startsWith(path) && td.fullyQualifiedName.substring(path.length).lastIndexOf('.') <= 0) } - - def tasks(taskDefs: Array[TaskDef]): Array[Task] = - for { + + def tasks(taskDefs: Array[TaskDef]): Array[Task] = + for { taskDef <- if (wildcard.isEmpty && membersOnly.isEmpty) taskDefs else (filterWildcard(wildcard, taskDefs) ++ filterMembersOnly(membersOnly, taskDefs)).distinct task = createTask(taskDef) if task.shouldDiscover } yield task - + def done = { if (!isDone.getAndSet(true)) { @@ -772,7 +773,7 @@ class Framework extends SbtFramework { } val duration = System.currentTimeMillis - runStartTime - val summary = new Summary(summaryCounter.testsSucceededCount.get, summaryCounter.testsFailedCount.get, summaryCounter.testsIgnoredCount.get, summaryCounter.testsPendingCount.get, + val summary = new Summary(summaryCounter.testsSucceededCount.get, summaryCounter.testsFailedCount.get, summaryCounter.testsIgnoredCount.get, summaryCounter.testsPendingCount.get, summaryCounter.testsCanceledCount.get, summaryCounter.suitesCompletedCount.get, summaryCounter.suitesAbortedCount.get, summaryCounter.scopesPendingCount.get) dispatchReporter(RunCompleted(tracker.nextOrdinal(), Some(duration), Some(summary))) dispatchReporter.dispatchDisposeAndWaitUntilDone() @@ -791,7 +792,7 @@ class Framework extends SbtFramework { presentReminderWithFullStackTraces, presentReminderWithoutCanceledTests, presentFilePathname - ) + ) fragments.map(_.toPossiblyColoredText(presentInColor)).mkString("\n") } else @@ -820,63 +821,63 @@ class Framework extends SbtFramework { } class Skeleton extends Runnable { - + val server = new ServerSocket(0) lazy val socket = new AtomicReference(server.accept()) lazy val is = new AtomicReference(new SkeletonObjectInputStream(socket.get.getInputStream, getClass.getClassLoader)) - + def run(): Unit = { try { (new React(server)).tryReact(0) - } + } finally { - is.get.close() + is.get.close() socket.get.close() } } - + class React(server: ServerSocket) { - @annotation.tailrec - final def react(): Unit = { + @annotation.tailrec + final def react(): Unit = { val event = is.get.readObject event match { case e: TestStarting => - dispatchReporter(e) + dispatchReporter(e) react() - case e: TestSucceeded => - dispatchReporter(e) + case e: TestSucceeded => + dispatchReporter(e) summaryCounter.incrementTestsSucceededCount() react() - case e: TestFailed => - dispatchReporter(e) + case e: TestFailed => + dispatchReporter(e) summaryCounter.incrementTestsFailedCount() react() - case e: TestIgnored => + case e: TestIgnored => dispatchReporter(e) summaryCounter.incrementTestsIgnoredCount() react() - case e: TestPending => + case e: TestPending => dispatchReporter(e) summaryCounter.incrementTestsPendingCount() react() - case e: TestCanceled => + case e: TestCanceled => dispatchReporter(e) summaryCounter.incrementTestsCanceledCount() react() - case e: SuiteStarting => + case e: SuiteStarting => dispatchReporter(e) react() - case e: SuiteCompleted => + case e: SuiteCompleted => dispatchReporter(e) summaryCounter.incrementSuitesCompletedCount() react() - case e: SuiteAborted => + case e: SuiteAborted => dispatchReporter(e) summaryCounter.incrementSuitesAbortedCount() react() case e: ScopeOpened => dispatchReporter(e); react() case e: ScopeClosed => dispatchReporter(e); react() - case e: ScopePending => + case e: ScopePending => dispatchReporter(e) summaryCounter.incrementScopesPendingCount() react() @@ -888,17 +889,17 @@ class Framework extends SbtFramework { case e: RunCompleted => // Sub-process completed, just let the thread terminate case e: RunStopped => dispatchReporter(e) case e: RunAborted => dispatchReporter(e) - } + } } @annotation.tailrec - final def tryReact(count: Int): Unit = + final def tryReact(count: Int): Unit = if (count < 3) try { - react() + react() } catch { - case t: IOException => + case t: IOException => // Restart server socket println(Resources.unableToReadSerializedEvent) is.get.close() @@ -909,13 +910,13 @@ class Framework extends SbtFramework { else { println(Resources.unableToContinueRun) System.exit(-1) - } + } } - + def host: String = server.getLocalSocketAddress.toString def port: Int = server.getLocalPort } - + val skeleton = new Skeleton() val thread = new Thread(skeleton) thread.start() @@ -973,20 +974,20 @@ class Framework extends SbtFramework { membersOnlyArgs, wildcardArgs, testNGArgs, - suffixes, - chosenStyles, - spanScaleFactors, + suffixes, + chosenStyles, + spanScaleFactors, testSortingReporterTimeouts, slowpokeArgs, seedArgs ) = parseArgs(args) - + if (!runpathArgs.isEmpty) throw new IllegalArgumentException("Specifying a runpath (-R ) is not supported when running ScalaTest from sbt.") - + if (!againArgs.isEmpty) throw new IllegalArgumentException("Run again (-A) is not supported when running ScalaTest from sbt; Please use sbt's test-quick instead.") - + if (!junitArgs.isEmpty) throw new IllegalArgumentException("Running JUnit tests (-j ) is not supported when running ScalaTest from sbt.") @@ -997,12 +998,12 @@ class Framework extends SbtFramework { throw new IllegalArgumentException("Discovery suffixes (-q) is not supported when running ScalaTest from sbt; Please use sbt's test-only or test filter instead.") val testSortingReporterTimeout = Span(parseDoubleArgument(testSortingReporterTimeouts, "-T", Suite.defaultTestSortingReporterTimeoutInSeconds), Seconds) - + val propertiesMap = parsePropertiesArgsIntoMap(propertiesArgs) val chosenStyleSet: Set[String] = parseChosenStylesIntoChosenStyleSet(chosenStyles, "-y") if (propertiesMap.isDefinedAt(Suite.CHOSEN_STYLES)) throw new IllegalArgumentException("Property name '" + Suite.CHOSEN_STYLES + "' is used by ScalaTest, please choose other property name.") - val configMap: ConfigMap = + val configMap: ConfigMap = if (chosenStyleSet.isEmpty) propertiesMap else @@ -1010,7 +1011,7 @@ class Framework extends SbtFramework { if (chosenStyleSet.nonEmpty) println(Resources.deprecatedChosenStyleWarning) - + val tagsToInclude: Set[String] = parseCompoundArgIntoSet(tagsToIncludeArgs, "-n") val tagsToExclude: Set[String] = parseCompoundArgIntoSet(tagsToExcludeArgs, "-l") val membersOnly: List[String] = parseSuiteArgsIntoNameStrings(membersOnlyArgs, "-m") @@ -1041,8 +1042,8 @@ class Framework extends SbtFramework { val (stderrArgs, others) = nonStdoutArgs.partition(_.startsWith("-e")) (stdoutArgs.take(1), stderrArgs.take(1), others) } - - val fullReporterConfigurations: ReporterConfigurations = + + val fullReporterConfigurations: ReporterConfigurations = if (remoteArgs.isEmpty) { // Creating the normal/main runner, should create reporters as specified by args. // If no reporters specified, just give them a default stdout reporter @@ -1069,16 +1070,16 @@ class Framework extends SbtFramework { presentFilePathname, presentJson, configSet - ) = + ) = fullReporterConfigurations.standardOutReporterConfiguration match { case Some(stdoutConfig) => val configSet = stdoutConfig.configSet ( - true, + true, configSet.contains(PresentAllDurations), !configSet.contains(PresentWithoutColor) && !sbtNoFormat, configSet.contains(PresentShortStackTraces) || configSet.contains(PresentFullStackTraces), - configSet.contains(PresentFullStackTraces), + configSet.contains(PresentFullStackTraces), configSet.contains(PresentUnformatted), configSet.exists { ele => ele == PresentReminderWithoutStackTraces || ele == PresentReminderWithShortStackTraces || ele == PresentReminderWithFullStackTraces @@ -1097,7 +1098,7 @@ class Framework extends SbtFramework { // able to get the Array[Logger] to create SbtInfoLoggerReporter. (!remoteArgs.isEmpty || reporterArgs.isEmpty, false, !sbtNoFormat, false, false, false, false, false, false, false, false, false, Set.empty[ReporterConfigParam]) } - + //val reporterConfigs = fullReporterConfigurations.copy(standardOutReporterConfiguration = None) // If there's a graphic reporter, we need to leave it out of // reporterSpecs, because we want to pass all reporterSpecs except @@ -1124,12 +1125,12 @@ class Framework extends SbtFramework { testClassLoader, tagsToInclude, tagsToExclude, - membersOnly, + membersOnly, wildcard, autoSelectors, configMap, reporterConfigs, - useStdout, + useStdout, presentAllDurations, presentInColor, presentShortStackTraces, @@ -1149,66 +1150,66 @@ class Framework extends SbtFramework { testSortingReporterTimeout ) } - + private case class ScalaTestSbtEvent( - fullyQualifiedName: String, - fingerprint: Fingerprint, - selector: Selector, - status: SbtStatus, - throwable: OptionalThrowable, + fullyQualifiedName: String, + fingerprint: Fingerprint, + selector: Selector, + status: SbtStatus, + throwable: OptionalThrowable, duration: Long) extends SbtEvent - + private class SbtReporter(suiteId: String, fullyQualifiedName: String, fingerprint: Fingerprint, eventHandler: EventHandler, report: Reporter, summaryCounter: SummaryCounter) extends Reporter { - + import org.scalatest.events._ - + private def getTestSelector(eventSuiteId: String, testName: String) = { if (suiteId == eventSuiteId) new TestSelector(testName) else new NestedTestSelector(eventSuiteId, testName) } - + private def getSuiteSelector(eventSuiteId: String) = { if (suiteId == eventSuiteId) new SuiteSelector else new NestedSuiteSelector(eventSuiteId) } - - private def getOptionalThrowable(throwable: Option[Throwable]): OptionalThrowable = + + private def getOptionalThrowable(throwable: Option[Throwable]): OptionalThrowable = throwable match { case Some(t) => new OptionalThrowable(t) case None => new OptionalThrowable } - + override def apply(event: Event): Unit = { report(event) event match { // the results of running an actual test - case t: TestPending => + case t: TestPending => summaryCounter.incrementTestsPendingCount() eventHandler.handle(ScalaTestSbtEvent(fullyQualifiedName, fingerprint, getTestSelector(t.suiteId, t.testName), SbtStatus.Pending, new OptionalThrowable, t.duration.getOrElse(0))) - case t: TestFailed => + case t: TestFailed => summaryCounter.incrementTestsFailedCount() eventHandler.handle(ScalaTestSbtEvent(fullyQualifiedName, fingerprint, getTestSelector(t.suiteId, t.testName), SbtStatus.Failure, getOptionalThrowable(t.throwable), t.duration.getOrElse(0))) - case t: TestSucceeded => + case t: TestSucceeded => summaryCounter.incrementTestsSucceededCount() eventHandler.handle(ScalaTestSbtEvent(fullyQualifiedName, fingerprint, getTestSelector(t.suiteId, t.testName), SbtStatus.Success, new OptionalThrowable, t.duration.getOrElse(0))) - case t: TestIgnored => + case t: TestIgnored => summaryCounter.incrementTestsIgnoredCount() eventHandler.handle(ScalaTestSbtEvent(fullyQualifiedName, fingerprint, getTestSelector(t.suiteId, t.testName), SbtStatus.Ignored, new OptionalThrowable, -1)) case t: TestCanceled => summaryCounter.incrementTestsCanceledCount() eventHandler.handle(ScalaTestSbtEvent(fullyQualifiedName, fingerprint, getTestSelector(t.suiteId, t.testName), SbtStatus.Canceled, new OptionalThrowable, t.duration.getOrElse(0))) - case t: SuiteCompleted => + case t: SuiteCompleted => summaryCounter.incrementSuitesCompletedCount() - case t: SuiteAborted => + case t: SuiteAborted => summaryCounter.incrementSuitesAbortedCount() eventHandler.handle(ScalaTestSbtEvent(fullyQualifiedName, fingerprint, getSuiteSelector(t.suiteId), SbtStatus.Error, getOptionalThrowable(t.throwable), t.duration.getOrElse(0))) - case t: ScopePending => + case t: ScopePending => summaryCounter.incrementScopesPendingCount() - case _ => + case _ => } } } diff --git a/jvm/scalatest-test/src/test/scala/org/scalatest/tools/FrameworkSuite.scala b/jvm/scalatest-test/src/test/scala/org/scalatest/tools/FrameworkSuite.scala index ebe10bb7f3..ccd12a2988 100644 --- a/jvm/scalatest-test/src/test/scala/org/scalatest/tools/FrameworkSuite.scala +++ b/jvm/scalatest-test/src/test/scala/org/scalatest/tools/FrameworkSuite.scala @@ -43,7 +43,7 @@ class FrameworkSuite extends AnyFunSuite { private var ignoredEvents = List[Event]() private var pendingEvents = List[Event]() private var canceledEvents = List[Event]() - + override def handle(event: Event): Unit = { event.status match { case Status.Success => successEvents ::= event @@ -55,7 +55,7 @@ class FrameworkSuite extends AnyFunSuite { case Status.Canceled => canceledEvents ::= event } } - + def errorEventsReceived = errorEvents.reverse def failureEventsReceived = failureEvents.reverse def skippedEventsReceived = skippedEvents.reverse @@ -64,15 +64,15 @@ class FrameworkSuite extends AnyFunSuite { def pendingEventsReceived = pendingEvents.reverse def canceledEventsReceived = canceledEvents.reverse } - + class TestLogger extends Logger { - + private var errorList = List[String]() private var warnList = List[String]() private var infoList = List[String]() private var debugList = List[String]() private var traceList = List[Throwable]() - + def ansiCodesSupported = false def error(msg: String): Unit = { errorList ::= msg @@ -89,7 +89,7 @@ class FrameworkSuite extends AnyFunSuite { def trace(t: Throwable): Unit = { traceList ::= t } - + def errorReceived = errorList.reverse def warnReceived = warnList.reverse def infoReceived = infoList.reverse @@ -100,7 +100,7 @@ class FrameworkSuite extends AnyFunSuite { test("framework name") { assert(new Framework().name === "ScalaTest") } - + test("fingerprints contains 2 test fingerprints, they are SubclassFingerprint for org.scalatest.Suite and AnnotatedFingerprint for org.scalatest.WrapWith") { val framework = new Framework val fingerprints = framework.fingerprints @@ -111,33 +111,33 @@ class FrameworkSuite extends AnyFunSuite { assert(testFingerprint.isModule === false) assert(testFingerprint.superclassName === "org.scalatest.Suite") assert(testFingerprint.requireNoArgConstructor === true) - - val annotatedFingerprint = + + val annotatedFingerprint = fingerprints(1).asInstanceOf[sbt.testing.AnnotatedFingerprint] assert(annotatedFingerprint.isModule === false) assert(annotatedFingerprint.annotationName === "org.scalatest.WrapWith") } - + val testClassLoader = getClass.getClassLoader val subClassFingerprint = new sbt.testing.SubclassFingerprint { def superclassName = "org.scalatest.Suite" def isModule = false def requireNoArgConstructor = true } - + val framework = new Framework - val subclassFingerprint = + val subclassFingerprint = new SubclassFingerprint { def superclassName = "org.scalatest.Suite" def isModule = false def requireNoArgConstructor = true } - val annotatedFingerprint = + val annotatedFingerprint = new AnnotatedFingerprint { def annotationName = "org.scalatest.WrapWith" def isModule = false } - + def assertSuiteSuccessEvent(event: Event, suiteClassName: String, testName: String, fingerprint: Fingerprint): Unit = { assert(Status.Success === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -146,13 +146,13 @@ class FrameworkSuite extends AnyFunSuite { assert(!event.throwable.isDefined) val selector = event.selector selector match { - case testSelector: TestSelector => + case testSelector: TestSelector => assert(testName === testSelector.testName) - case _ => + case _ => fail("Expected to get TestSelector, but got: " + selector.getClass.getName) } } - + def assertNestedSuiteSuccessEvent(event: Event, suiteClassName: String, suiteId:String, testName: String, fingerprint: Fingerprint): Unit = { assert(Status.Success === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -161,14 +161,14 @@ class FrameworkSuite extends AnyFunSuite { assert(!event.throwable.isDefined) val selector = event.selector selector match { - case nestedTestSelector: NestedTestSelector => + case nestedTestSelector: NestedTestSelector => assert(suiteId === nestedTestSelector.suiteId) assert(testName === nestedTestSelector.testName) - case _ => + case _ => fail("Expected to get NestedTestSelector, but got: " + selector.getClass.getName) } } - + def assertSuiteFailureEvent(event: Event, suiteClassName: String, testName: String, fingerprint: Fingerprint): Unit = { assert(Status.Failure === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -177,13 +177,13 @@ class FrameworkSuite extends AnyFunSuite { assert(event.throwable.isDefined) val selector = event.selector selector match { - case testSelector: TestSelector => + case testSelector: TestSelector => assert(testName === testSelector.testName) - case _ => + case _ => fail("Expected to get TestSelector, but got: " + selector.getClass.getName) } } - + def assertNestedSuiteFailureEvent(event: Event, suiteClassName: String, suiteId:String, testName: String, fingerprint: Fingerprint): Unit = { assert(Status.Failure === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -192,14 +192,14 @@ class FrameworkSuite extends AnyFunSuite { assert(event.throwable.isDefined) val selector = event.selector selector match { - case nestedTestSelector: NestedTestSelector => + case nestedTestSelector: NestedTestSelector => assert(suiteId === nestedTestSelector.suiteId) assert(testName === nestedTestSelector.testName) - case _ => + case _ => fail("Expected to get NestedTestSelector, but got: " + selector.getClass.getName) } } - + def assertSuiteErrorEvent(event: Event, suiteClassName: String, fingerprint: Fingerprint): Unit = { assert(Status.Error === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -208,13 +208,13 @@ class FrameworkSuite extends AnyFunSuite { assert(event.throwable.isDefined) val selector = event.selector selector match { - case suiteSelector: SuiteSelector => + case suiteSelector: SuiteSelector => // Nothing more to check, just make sure it is SuiteSelector. - case _ => + case _ => fail("Expected to get TestSelector, but got: " + selector.getClass.getName) } } - + def assertNestedSuiteErrorEvent(event: Event, suiteClassName: String, suiteId:String, fingerprint: Fingerprint): Unit = { assert(Status.Error === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -223,13 +223,13 @@ class FrameworkSuite extends AnyFunSuite { assert(event.throwable.isDefined) val selector = event.selector selector match { - case nestedSuiteSelector: NestedSuiteSelector => + case nestedSuiteSelector: NestedSuiteSelector => assert(suiteId === nestedSuiteSelector.suiteId) - case _ => + case _ => fail("Expected to get NestedTestSelector, but got: " + selector.getClass.getName) } } - + def assertSuiteSkippedEvent(event: Event, suiteClassName: String, testName: String, fingerprint: Fingerprint): Unit = { assert(Status.Skipped === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -238,13 +238,13 @@ class FrameworkSuite extends AnyFunSuite { assert(!event.throwable.isDefined) val selector = event.selector selector match { - case testSelector: TestSelector => + case testSelector: TestSelector => assert(testName === testSelector.testName) - case _ => + case _ => fail("Expected to get TestSelector, but got: " + selector.getClass.getName) } } - + def assertSuiteIgnoredEvent(event: Event, suiteClassName: String, testName: String, fingerprint: Fingerprint): Unit = { assert(Status.Ignored === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -253,13 +253,13 @@ class FrameworkSuite extends AnyFunSuite { assert(!event.throwable.isDefined) val selector = event.selector selector match { - case testSelector: TestSelector => + case testSelector: TestSelector => assert(testName === testSelector.testName) - case _ => + case _ => fail("Expected to get TestSelector, but got: " + selector.getClass.getName) } } - + def assertSuitePendingEvent(event: Event, suiteClassName: String, testName: String, fingerprint: Fingerprint): Unit = { assert(Status.Pending === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -268,13 +268,13 @@ class FrameworkSuite extends AnyFunSuite { assert(!event.throwable.isDefined) val selector = event.selector selector match { - case testSelector: TestSelector => + case testSelector: TestSelector => assert(testName === testSelector.testName) - case _ => + case _ => fail("Expected to get TestSelector, but got: " + selector.getClass.getName) } } - + def assertSuiteCanceledEvent(event: Event, suiteClassName: String, testName: String, fingerprint: Fingerprint): Unit = { assert(Status.Canceled === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -283,13 +283,13 @@ class FrameworkSuite extends AnyFunSuite { assert(!event.throwable.isDefined) val selector = event.selector selector match { - case testSelector: TestSelector => + case testSelector: TestSelector => assert(testName === testSelector.testName) - case _ => + case _ => fail("Expected to get TestSelector, but got: " + selector.getClass.getName) } } - + def assertNestedSuiteSkippedEvent(event: Event, suiteClassName: String, suiteId:String, testName: String, fingerprint: Fingerprint): Unit = { assert(Status.Skipped === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -298,14 +298,14 @@ class FrameworkSuite extends AnyFunSuite { assert(!event.throwable.isDefined) val selector = event.selector selector match { - case nestedTestSelector: NestedTestSelector => + case nestedTestSelector: NestedTestSelector => assert(suiteId === nestedTestSelector.suiteId) assert(testName === nestedTestSelector.testName) - case _ => + case _ => fail("Expected to get NestedTestSelector, but got: " + selector.getClass.getName) } } - + def assertNestedSuiteIgnoredEvent(event: Event, suiteClassName: String, suiteId:String, testName: String, fingerprint: Fingerprint): Unit = { assert(Status.Ignored === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -314,14 +314,14 @@ class FrameworkSuite extends AnyFunSuite { assert(!event.throwable.isDefined) val selector = event.selector selector match { - case nestedTestSelector: NestedTestSelector => + case nestedTestSelector: NestedTestSelector => assert(suiteId === nestedTestSelector.suiteId) assert(testName === nestedTestSelector.testName) - case _ => + case _ => fail("Expected to get NestedTestSelector, but got: " + selector.getClass.getName) } } - + def assertNestedSuitePendingEvent(event: Event, suiteClassName: String, suiteId:String, testName: String, fingerprint: Fingerprint): Unit = { assert(Status.Pending === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -330,14 +330,14 @@ class FrameworkSuite extends AnyFunSuite { assert(!event.throwable.isDefined) val selector = event.selector selector match { - case nestedTestSelector: NestedTestSelector => + case nestedTestSelector: NestedTestSelector => assert(suiteId === nestedTestSelector.suiteId) assert(testName === nestedTestSelector.testName) - case _ => + case _ => fail("Expected to get NestedTestSelector, but got: " + selector.getClass.getName) } } - + def assertNestedSuiteCanceledEvent(event: Event, suiteClassName: String, suiteId:String, testName: String, fingerprint: Fingerprint): Unit = { assert(Status.Canceled === event.status) assert(suiteClassName === event.fullyQualifiedName) @@ -346,19 +346,19 @@ class FrameworkSuite extends AnyFunSuite { assert(!event.throwable.isDefined) val selector = event.selector selector match { - case nestedTestSelector: NestedTestSelector => + case nestedTestSelector: NestedTestSelector => assert(suiteId === nestedTestSelector.suiteId) assert(testName === nestedTestSelector.testName) - case _ => + case _ => fail("Expected to get NestedTestSelector, but got: " + selector.getClass.getName) } } - + test("ScalaTestRunner.task should return task that run whole suite when fullyQualifiedName = valid class name, explicitlySpecified = false and selectors = Array(SuiteSelector)") { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) try { val testEventHandler = new TestEventHandler - + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array(new SuiteSelector)))) assert(tasks.size === 1) val task = tasks(0) @@ -382,7 +382,7 @@ class FrameworkSuite extends AnyFunSuite { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) try { val testEventHandler = new TestEventHandler - + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.DoNotDiscoverSuite", subclassFingerprint, false, Array(new SuiteSelector)))) assert(tasks.size === 0) } @@ -390,7 +390,7 @@ class FrameworkSuite extends AnyFunSuite { runner.done() } } - + test("When suite is neither subclass of org.scalatest.Suite or annotated with WrapWith and explicitlySpecified is true, IllegalArgumentException will be thrown when task executes") { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) try { @@ -405,7 +405,7 @@ class FrameworkSuite extends AnyFunSuite { runner.done() } } - + test("When suite is neither subclass of org.scalatest.Suite or annotated with WrapWith and explicitlySpecified is false, no task will be returned") { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) try { @@ -432,12 +432,12 @@ class FrameworkSuite extends AnyFunSuite { runner.done() } } - + test("Nested suites will be executed in task(fullyQualifiedName: String, fingerprint: Fingerprint), no nested task will be returned") { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) try { val testEventHandler = new TestEventHandler - + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SuiteWithNestedSuites", subclassFingerprint, false, Array(new SuiteSelector)))) assert(tasks.size === 1) val task = tasks(0) @@ -462,12 +462,12 @@ class FrameworkSuite extends AnyFunSuite { runner.done() } } - + test("Ignore, pending, failed, canceled, suite aborted events should be translated and reported correctly for the suite and its nested suites") { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) try { val testEventHandler = new TestEventHandler - + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SuiteWithFailedSkippedTests", subclassFingerprint, false, Array(new SuiteSelector)))) assert(tasks.size === 1) val task = tasks(0) @@ -520,7 +520,7 @@ class FrameworkSuite extends AnyFunSuite { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) try { val testEventHandler = new TestEventHandler - + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array(new SuiteSelector())))) val task = tasks(0) task.execute(testEventHandler, Array(new TestLogger)) @@ -536,7 +536,7 @@ class FrameworkSuite extends AnyFunSuite { runner.done() } } - + test("SuiteSelector should select and run test(s) in selected suite when it is explicitly specified, even when the selected suite is annotated with @DoNotDiscover") { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) try { @@ -562,7 +562,7 @@ class FrameworkSuite extends AnyFunSuite { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) try { val testEventHandler = new TestEventHandler - val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array(new TestSelector("test 1"), new TestSelector("test 3"))), + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array(new TestSelector("test 1"), new TestSelector("test 3"))), new TaskDef("org.scalatest.tools.scalasbt.SuiteWithNestedSuites", subclassFingerprint, false, Array(new TestSelector("test 2"))))) assert(tasks.size === 2) val task = tasks(0) @@ -574,7 +574,7 @@ class FrameworkSuite extends AnyFunSuite { assert(testEventHandler.errorEventsReceived.length === 0) assert(testEventHandler.failureEventsReceived.length === 0) assert(testEventHandler.skippedEventsReceived.length === 0) - + val testEventHandler2 = new TestEventHandler val task2 = tasks(1) val task2NestedSuites = task2.execute(testEventHandler2, Array(new TestLogger)) @@ -691,18 +691,18 @@ class FrameworkSuite extends AnyFunSuite { runner.done() } } - + test("NestedSuiteSelector should select and run test(s) in selected nested suite when it is explicitly specified, even if the selected nested suite is annotated with @DoNotDiscover") { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) try { val testEventHandler = new TestEventHandler - + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SuiteWithNestedSuites", subclassFingerprint, true, Array(new NestedSuiteSelector("nested 1"))))) assert(tasks.size == 1) val task = tasks(0) val nestedTasks = task.execute(testEventHandler, Array(new TestLogger)) assert(nestedTasks.size == 0) - + val successEvents = testEventHandler.successEventsReceived assert(successEvents.length == 3) assertNestedSuiteSuccessEvent(successEvents(0), "org.scalatest.tools.scalasbt.SuiteWithNestedSuites", "nested 1", "nested 1 test 1", subclassFingerprint) @@ -722,13 +722,13 @@ class FrameworkSuite extends AnyFunSuite { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) try { val testEventHandler = new TestEventHandler - + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SuiteWithNestedSuites", subclassFingerprint, false, Array(new NestedSuiteSelector("nested 1"))))) assert(tasks.size === 1) val task = tasks(0) val nestedTasks = task.execute(testEventHandler, Array(new TestLogger)) assert(nestedTasks.size === 0) - + val successEvents = testEventHandler.successEventsReceived assert(successEvents.length === 3) assertNestedSuiteSuccessEvent(successEvents(0), "org.scalatest.tools.scalasbt.SuiteWithNestedSuites", "nested 1", "nested 1 test 1", subclassFingerprint) @@ -746,10 +746,10 @@ class FrameworkSuite extends AnyFunSuite { test("NestedTestSelector should select and run selected test(s) in selected nested suite when it is explicitly specified, even if the selected nested suite is annotated with @DoNotDiscover") { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) - + try { val testEventHandler = new TestEventHandler - + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SuiteWithNestedSuites", subclassFingerprint, false, Array(new NestedTestSelector("nested 1", "nested 1 test 1"), new NestedTestSelector("nested 2", "nested 2 test 3"))))) assert(tasks.size == 1) val task = tasks(0) @@ -774,7 +774,7 @@ class FrameworkSuite extends AnyFunSuite { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) try { val testEventHandler = new TestEventHandler - + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SuiteWithNestedSuites", subclassFingerprint, false, Array(new NestedTestSelector("nested 1", "nested 1 test 1"), new NestedTestSelector("nested 2", "nested 2 test 3"))))) assert(tasks.size === 1) val task = tasks(0) @@ -797,7 +797,7 @@ class FrameworkSuite extends AnyFunSuite { test("ScalaTestRunner should return summary when 'done' is called, and throw IllegalStateException if 'done' method is called twice.") { val runner = framework.runner(Array("-oW"), Array.empty, testClassLoader) - + try { val testLogger = new TestLogger val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array(new SuiteSelector())))) @@ -827,7 +827,7 @@ class FrameworkSuite extends AnyFunSuite { test("ScalaTestRunner using -oWI should return summary that contains failed and canceled test reminder when 'done' is called") { val runner = framework.runner(Array("-oWI"), Array.empty, testClassLoader) - + try { val testLogger = new TestLogger val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SuiteWithFailedCanceledTests", subclassFingerprint, false, Array(new SuiteSelector())))) @@ -862,7 +862,7 @@ class FrameworkSuite extends AnyFunSuite { test("ScalaTestRunner using -oWIK should return summary that contains failed test reminder only (without canceled test) when 'done' is called") { val runner = framework.runner(Array("-oWIK"), Array.empty, testClassLoader) - + try { val testLogger = new TestLogger val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SuiteWithFailedCanceledTests", subclassFingerprint, false, Array(new SuiteSelector())))) @@ -906,7 +906,7 @@ class FrameworkSuite extends AnyFunSuite { runner.done() } } - + test("ScalaTest Task's tags method should return 'network' when suite class is annotated with @Network") { val runner = framework.runner(Array("-oW"), Array.empty, testClassLoader) try { @@ -954,26 +954,26 @@ class FrameworkSuite extends AnyFunSuite { runner.done() } } - + test("ScalaTest Task's taskDef method should return TaskDef that defines the task") { val runner = framework.runner(Array.empty, Array.empty, testClassLoader) - + try { val testEventHandler = new TestEventHandler val suiteSelector = new SuiteSelector(); - val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array.empty), - new TaskDef("org.scalatest.tools.scalasbt.DoNotDiscoverSuite", subclassFingerprint, false, Array(suiteSelector)), + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array.empty), + new TaskDef("org.scalatest.tools.scalasbt.DoNotDiscoverSuite", subclassFingerprint, false, Array(suiteSelector)), new TaskDef("org.scalatest.tools.scalasbt.DoNotDiscoverSuite", subclassFingerprint, true, Array(suiteSelector)))) - + assert(tasks.length === 2) - + val task1 = tasks(0) val taskDef1 = task1.taskDef assert(taskDef1.fullyQualifiedName === "org.scalatest.tools.scalasbt.SampleSuite") assert(taskDef1.fingerprint === subclassFingerprint) assert(taskDef1.explicitlySpecified === false) assert(taskDef1.selectors.length === 0) - + val task2 = tasks(1) val taskDef2 = task2.taskDef assert(taskDef2.fullyQualifiedName === "org.scalatest.tools.scalasbt.DoNotDiscoverSuite") @@ -990,10 +990,10 @@ class FrameworkSuite extends AnyFunSuite { test("-l argument can be used to exclude test") { val runner = framework.runner(Array("-l", "org.scalatest.tools.scalasbt.SampleSuite.SlowTest"), Array.empty, testClassLoader) - + try { val testEventHandler = new TestEventHandler - + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array(new SuiteSelector)))) assert(tasks.size === 1) val task = tasks(0) @@ -1010,12 +1010,12 @@ class FrameworkSuite extends AnyFunSuite { runner.done() } } - + test("-n argument can be used to include test") { val runner = framework.runner(Array("-n", "org.scalatest.tools.scalasbt.SampleSuite.SlowTest"), Array.empty, testClassLoader) try { val testEventHandler = new TestEventHandler - + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array(new SuiteSelector)))) assert(tasks.size === 1) val task = tasks(0) @@ -1031,13 +1031,13 @@ class FrameworkSuite extends AnyFunSuite { runner.done() } } - + test("-w should execute suites that match the specified package and its sub packages") { val runner = framework.runner(Array("-w", "org.scalatest.tools"), Array.empty, testClassLoader) try { val testEventHandler = new TestEventHandler - val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array(new SuiteSelector)), - new TaskDef("org.scalatest.tools.FrameworkSuite", subclassFingerprint, false, Array(new SuiteSelector)), + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array(new SuiteSelector)), + new TaskDef("org.scalatest.tools.FrameworkSuite", subclassFingerprint, false, Array(new SuiteSelector)), new TaskDef("org.scalatest.SuiteSuite", subclassFingerprint, false, Array(new SuiteSelector)))) assert(tasks.size === 2) } @@ -1049,8 +1049,8 @@ class FrameworkSuite extends AnyFunSuite { test("-m should execute suites that match the specified package and not its sub packages") { val runner = framework.runner(Array("-m", "org.scalatest.tools"), Array.empty, testClassLoader) try { - val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array(new SuiteSelector)), - new TaskDef("org.scalatest.tools.FrameworkSuite", subclassFingerprint, false, Array(new SuiteSelector)), + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array(new SuiteSelector)), + new TaskDef("org.scalatest.tools.FrameworkSuite", subclassFingerprint, false, Array(new SuiteSelector)), new TaskDef("org.scalatest.SuiteSuite", subclassFingerprint, false, Array(new SuiteSelector)))) assert(tasks.size === 1) val runner2 = framework.runner(Array("-m", "org.scalatest.concurrent"), Array.empty, testClassLoader) @@ -1066,7 +1066,7 @@ class FrameworkSuite extends AnyFunSuite { runner.done() } } - + // Now in 0.13.0-RC4 when there are 2 TaskDef with same class name different fingerprint, only one of it will be passed in. // We can't rely on fingerprint for this check anymore. /*test("a suite should be filtered out when fingerprint is subclassFingerprint and it is not accessible, even though it is annotated with @WrapWith") { @@ -1074,14 +1074,14 @@ class FrameworkSuite extends AnyFunSuite { val tasks = runner.tasks(Array(new TaskDef("org.scalatest.SavesConfigMapSuite", subclassFingerprint, false, Array(new SuiteSelector)))) assert(tasks.size === 0) }*/ - + test("Framework.runner should throw IllegalArgumentException when -s is passed in") { val iae = intercept[IllegalArgumentException] { framework.runner(Array("-s", "org.scalatest.tools.scalasbt.SampleSuite"), Array.empty, testClassLoader) } assert(iae.getMessage === "Specifying a suite (-s ) or nested suite (-i ) is not supported when running ScalaTest from sbt; Please use sbt's test-only instead.") } - + test("Framework.runner should throw IllegalArgumentException when -j is passed in") { val iae = intercept[IllegalArgumentException] { framework.runner(Array("-j", "org.scalatest.tools.scalasbt.SampleJUnitSuite"), Array.empty, testClassLoader) @@ -1095,7 +1095,7 @@ class FrameworkSuite extends AnyFunSuite { } assert(iae.getMessage === "Running TestNG tests (-b ) is not supported when running ScalaTest from sbt.") } - + test("Framework.runner should throw IllegalArgumentException when -P is passed in") { val iae = intercept[IllegalArgumentException] { framework.runner(Array("-P"), Array.empty, testClassLoader) @@ -1121,14 +1121,14 @@ class FrameworkSuite extends AnyFunSuite { } assert(iae.getMessage === "-P with negative or zero thread number is invalid, please pass in a positive thread number instead.") } - + test("Framework.runner should throw IllegalArgumentException when -PS is passed in") { val iae = intercept[IllegalArgumentException] { framework.runner(Array("-PS"), Array.empty, testClassLoader) } assert(iae.getMessage === "-PS is not supported when running ScalaTest from sbt, please use sbt parallel and logBuffered configuration instead.") } - + test("Framework.runner should throw IllegalArgumentException when -R is passed in") { val iae = intercept[IllegalArgumentException] { framework.runner(Array("-R"), Array.empty, testClassLoader) @@ -1142,7 +1142,7 @@ class FrameworkSuite extends AnyFunSuite { } assert(iae.getMessage === "Run again (-A) is not supported when running ScalaTest from sbt; Please use sbt's test-quick instead.") } - + test("Framework.runner should throw IllegalArgumentException when -q is passed in") { val iae = intercept[IllegalArgumentException] { framework.runner(Array("-q", "Spec"), Array.empty, testClassLoader) @@ -1154,13 +1154,13 @@ class FrameworkSuite extends AnyFunSuite { val runner = framework.runner(Array("-T", "100"), Array.empty, testClassLoader) runner.done() } - + private def makeSureDone(runners: Runner*)(fun: => Unit): Unit = { try { fun } finally { - runners.foreach { r => + runners.foreach { r => try { r.done() } @@ -1183,13 +1183,34 @@ class FrameworkSuite extends AnyFunSuite { val scalatestRunner = runner.asInstanceOf[org.scalatest.tools.Framework#ScalaTestRunner] scalatestRunner.done() scalatestRunner.dispatchReporter.reporters.find(_.isInstanceOf[EventRecordingReporter]) match { - case Some(recordingRep : EventRecordingReporter) => + case Some(recordingRep : EventRecordingReporter) => assert(recordingRep.testSucceededEventsReceived.size === 3) case _ => fail("Expected to find EventRecordingReporter, but not found.") } } } + test("should report SuiteAborted event when before function in BeforeAndAfterEachSuite in an async test throws RuntimeException") { + val runner = framework.runner(Array("-C", classOf[EventRecordingReporter].getName), Array.empty, testClassLoader) + makeSureDone(runner) { + val testEventHandler = new TestEventHandler + val tasks = runner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.BadBeforeAsyncSuite", subclassFingerprint, false, Array(new SuiteSelector)))) + val task = tasks(0) + val logger = new TestLogger + task.execute(testEventHandler, Array(logger)) + assert(runner.isInstanceOf[org.scalatest.tools.Framework#ScalaTestRunner]) + val scalatestRunner = runner.asInstanceOf[org.scalatest.tools.Framework#ScalaTestRunner] + scalatestRunner.done() + scalatestRunner.dispatchReporter.reporters.find(_.isInstanceOf[EventRecordingReporter]) match { + case Some(recordingRep: EventRecordingReporter) => + assert(recordingRep.suiteStartingEventsReceived.size === 1) + assert(recordingRep.suiteCompletedEventsReceived.size === 0) + assert(recordingRep.suiteAbortedEventsReceived.size === 1) + case _ => fail("Expected to find EventRecordingReporter, but not found.") + } + } + } + test("should fire SuiteAborted event when after function in BeforeAndAfter throws RuntimeException") { val runner = framework.runner(Array("-C", classOf[EventRecordingReporter].getName), Array.empty, testClassLoader) makeSureDone(runner) { @@ -1294,7 +1315,7 @@ class FrameworkSuite extends AnyFunSuite { } } } - + test("-W should cause AlertProvided to be fired", Flicker) { val runner = framework.runner(Array("-W", "1", "1", "-C", classOf[EventRecordingReporter].getName), Array.empty, testClassLoader) makeSureDone(runner) { @@ -1307,7 +1328,7 @@ class FrameworkSuite extends AnyFunSuite { val scalatestRunner = runner.asInstanceOf[org.scalatest.tools.Framework#ScalaTestRunner] scalatestRunner.done() scalatestRunner.dispatchReporter.reporters.find(_.isInstanceOf[EventRecordingReporter]) match { - case Some(recordingRep : EventRecordingReporter) => + case Some(recordingRep : EventRecordingReporter) => assert(recordingRep.testSucceededEventsReceived.size === 1) assert(recordingRep.alertProvidedEventsReceived.size > 0) case _ => fail("Expected to find EventRecordingReporter, but not found.") @@ -1321,7 +1342,7 @@ class FrameworkSuite extends AnyFunSuite { val subRunner = framework.runner(Array("-C", classOf[EventRecordingReporter].getName), remoteArgs, testClassLoader) makeSureDone(mainRunner, subRunner) { val testEventHandler = new TestEventHandler - + val tasks = subRunner.tasks(Array(new TaskDef("org.scalatest.tools.scalasbt.SampleSuite", subclassFingerprint, false, Array(new SuiteSelector)))) assert(tasks.size === 1) val task = tasks(0) @@ -1341,7 +1362,7 @@ class FrameworkSuite extends AnyFunSuite { subScalatestRunner.done() mainScalatestRunner.done() mainScalatestRunner.dispatchReporter.reporters.find(_.isInstanceOf[EventRecordingReporter]) match { - case Some(recordingRep : EventRecordingReporter) => + case Some(recordingRep : EventRecordingReporter) => assert(recordingRep.testSucceededEventsReceived.size === 3) assert(recordingRep.alertProvidedEventsReceived.size === 1) assert(recordingRep.noteProvidedEventsReceived.size === 1) diff --git a/jvm/scalatest-test/src/test/scala/org/scalatest/tools/scalasbt/BadBeforeAsyncSuite.scala b/jvm/scalatest-test/src/test/scala/org/scalatest/tools/scalasbt/BadBeforeAsyncSuite.scala new file mode 100644 index 0000000000..03b91798db --- /dev/null +++ b/jvm/scalatest-test/src/test/scala/org/scalatest/tools/scalasbt/BadBeforeAsyncSuite.scala @@ -0,0 +1,31 @@ +/* + * Copyright 2001-2015 Artima, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.scalatest.tools.scalasbt + +import org.scalatest._ +import org.scalatest.funsuite.AsyncFunSuite + +class BadBeforeAsyncSuite extends AsyncFunSuite with BeforeAndAfterEach { + + override protected def beforeEach(): Unit = { + throw new RuntimeException("oops!") + } + + test("test 1") { + succeed + } + +}