[Fix #8101] Reformat rake spec output to amplify signal and reduce …

…noise.

lib/rubocop/rspec/parallel_formatter.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+RSpec::Support.require_rspec_core 'formatters/base_text_formatter'
+RSpec::Support.require_rspec_core 'formatters/console_codes'
+
+module RuboCop
+  module RSpec
+    # RSpec formatter for use with running `rake spec` in parallel. This formatter
+    # removes much of the noise from RSpec so that only the important information
+    # will be surfaced by test-queue.
+    # It also adds metadata to the output in order to more easily find the text
+    # needed for outputting after the parallel run completes.
+    class ParallelFormatter < ::RSpec::Core::Formatters::BaseTextFormatter
+      ::RSpec::Core::Formatters.register self, :dump_pending, :dump_failures, :dump_summary
+
+      # Don't show pending tests
+      def dump_pending(*); end
+
+      # The BEGIN/END comments are used by `spec_runner.rake` to determine what
+      # output goes where in the final parallelized output, and should not be
+      # removed!
+      def dump_failures(notification)
+        return if notification.failure_notifications.empty?
+
+        output.puts '# FAILURES BEGIN'
+        notification.failure_notifications.each do |failure|
+          output.puts failure.fully_formatted('*', colorizer)
+        end
+        output.puts
+        output.puts '# FAILURES END'
+      end
+
+      def dump_summary(summary)
+        output_summary(summary)
+        output_rerun_commands(summary)
+      end
+
+      private
+
+      def colorizer
+        @colorizer ||= ::RSpec::Core::Formatters::ConsoleCodes
+      end
+
+      # The BEGIN/END comments are used by `spec_runner.rake` to determine what
+      # output goes where in the final parallelized output, and should not be
+      # removed!
+      def output_summary(summary)
+        output.puts '# SUMMARY BEGIN'
+        output.puts colorize_summary(summary)
+        output.puts '# SUMMARY END'
+      end
+
+      def colorize_summary(summary)
+        totals = totals(summary)
+
+        if summary.failure_count.positive? || summary.errors_outside_of_examples_count.positive?
+          colorizer.wrap(totals, ::RSpec.configuration.failure_color)
+        else
+          colorizer.wrap(totals, ::RSpec.configuration.success_color)
+        end
+      end
+
+      # The BEGIN/END comments are used by `spec_runner.rake` to determine what
+      # output goes where in the final parallelized output, and should not be
+      # removed!
+      def output_rerun_commands(summary)
+        output.puts '# RERUN BEGIN'
+        output.puts summary.colorized_rerun_commands.lines[3..-1].join
+        output.puts '# RERUN END'
+      end
+
+      def totals(summary)
+        output = pluralize(summary.example_count, 'example')
+        output += ", #{summary.pending_count} pending" if summary.pending_count.positive?
+        output += ", #{pluralize(summary.failure_count, 'failure')}"
+
+        if summary.errors_outside_of_examples_count.positive?
+          error_count = pluralize(summary.errors_outside_of_examples_count, 'error')
+          output += ", #{error_count} occurred outside of examples"
+        end
+
+        output
+      end
+
+      def pluralize(*args)
+        ::RSpec::Core::Formatters::Helpers.pluralize(*args)
+      end
+    end
+  end
+end

lib/rubocop/rspec/support.rb

@@ -6,6 +6,7 @@
 require_relative 'host_environment_simulation_helper'
 require_relative 'shared_contexts'
 require_relative 'expect_offense'
+require_relative 'parallel_formatter'
 
 RSpec.configure do |config|
   config.include CopHelper

spec/spec_helper.rb

@@ -78,3 +78,15 @@
     config.filter_run_excluding broken_on: :jruby
   end
 end
+
+module ::RSpec
+  module Core
+    class ExampleGroup
+      # Override `failure_count` from test-queue to prevent RSpec deprecation notice
+      # Treating `metadata[:execution_result]` as a hash is deprecated.
+      def self.failure_count
+        examples.map { |e| e.execution_result.status == 'failed' }.length
+      end
+    end
+  end
+end

tasks/spec_runner.rake

@@ -4,6 +4,14 @@ require 'rspec/core'
 require 'test_queue'
 require 'test_queue/runner/rspec'
 
+# Add `failed_examples` into `TestQueue::Worker` so we can keep
+# track of the output for re-running failed examples from RSpec.
+module TestQueue
+  class Worker
+    attr_accessor :failed_examples
+  end
+end
+
 module RuboCop
   # Helper for running specs with a temporary external encoding.
   # This is a bit risky, since strings defined before the block may have a
@@ -13,7 +21,7 @@ module RuboCop
   class SpecRunner
     attr_reader :rspec_args
 
-    def initialize(rspec_args = %w[spec], parallel: true,
+    def initialize(rspec_args = %w[spec --force-color], parallel: true,
                    external_encoding: 'UTF-8', internal_encoding: nil)
       @rspec_args = rspec_args
       @previous_external_encoding = Encoding.default_external
@@ -59,20 +67,58 @@ module RuboCop
     # `TestQueue::Runner::RSpec`, but modified so that it takes an argument
     # (an array of paths of specs to run) instead of relying on ARGV.
     class ParallelRunner < ::TestQueue::Runner
+      SUMMARY_REGEXP = /(?<=# SUMMARY BEGIN\n).*(?=\n# SUMMARY END)/m.freeze
+      FAILURE_OUTPUT_REGEXP = /(?<=# FAILURES BEGIN\n\n).*(?=# FAILURES END)/m.freeze
+      RERUN_REGEXP = /(?<=# RERUN BEGIN\n).+(?=\n# RERUN END)/m.freeze
+
       def initialize(rspec_args)
         super(Framework.new(rspec_args))
 
         @exit_when_done = false
+        @failure_count = 0
       end
 
       def run_worker(iterator)
         rspec = ::RSpec::Core::QueueRunner.new
         rspec.run_each(iterator).to_i
       end
 
+      # Override `TestQueue::Runner#worker_completed` to not output anything
+      # as it adds a lot of noise by default
+      def worker_completed(worker)
+        return if @aborting
+
+        @completed << worker
+      end
+
       def summarize_worker(worker)
-        worker.summary = worker.lines.grep(/\A\d+ examples?, /).first
-        worker.failure_output = worker.output[/^Failures:\n\n(.*)\n^Finished/m, 1]
+        worker.summary = worker.output[SUMMARY_REGEXP]
+        worker.failure_output = update_count(worker.output[FAILURE_OUTPUT_REGEXP])
+        worker.failed_examples = worker.output[RERUN_REGEXP]
+      end
+
+      def summarize_internal
+        ret = super
+
+        unless @failures.blank?
+          puts "==> Failed Examples\n\n"
+          puts @completed.map(&:failed_examples).compact.sort.join("\n")
+          puts
+        end
+
+        ret
+      end
+
+      private
+
+      def update_count(failures)
+        # The ParallelFormatter formatter doesn't try to count failures, but
+        # prefixes each with `*)`, so that they can be updated to count failures
+        # globally once all workers have completed.
+
+        return unless failures
+
+        failures.gsub('*)') { "#{@failure_count += 1})" }
       end
     end
 
@@ -96,7 +142,11 @@ module RuboCop
     class Framework < ::TestQueue::TestFramework::RSpec
       def initialize(rspec_args)
         super()
-        @rspec_args = rspec_args
+        formatter_args = %w[
+          --require ./lib/rubocop/rspec/parallel_formatter.rb
+          --format RuboCop::RSpec::ParallelFormatter
+        ]
+        @rspec_args = rspec_args.concat(formatter_args)
       end
 
       def all_suite_files