fixups

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,3 +5,4 @@ Thank you for your contribution!
 - [ ] Added tests.
 - [ ] Added an entry to the [Changelog](../blob/master/CHANGELOG.md) if the new 
   code introduces user-observable changes.
+- [ ] Update Readme.md when cli options are changed

CHANGELOG.md

@@ -8,7 +8,7 @@
 
 ### Added
 
-- `--fail-fast` options which stops all threads if one of them return not zero exit code. Which add possibility to stop whole suite if one test failed. Works if the option `--fail-fast` enabled for the rspec (passed to the test_options: `--test-options '--fail-fast'` or enabled at the .rspec_parallel file).  
+- `--fail-fast` stops all groups if one group fails. Can be used to stop all groups if one test failed by using `fail-fast` in the test-framework too (for example rspec via `--test-options '--fail-fast'` or in `.rspec_parallel`).  
 
 ### Fixed
 

Readme.md

@@ -192,7 +192,6 @@ Setup for non-rails
 
 Options are:
 <!-- copy output from bundle exec ./bin/parallel_test -h -->
-
     -n [PROCESSES]                   How many processes to use, default: available CPUs
     -p, --pattern [PATTERN]          run tests matching this regex pattern
         --exclude-pattern [PATTERN]  exclude tests matching this regex pattern
@@ -222,12 +221,14 @@ Options are:
         --ignore-tags [PATTERN]      When counting steps ignore scenarios with tags that match this pattern
         --nice                       execute test commands with low priority.
         --runtime-log [PATH]         Location of previously recorded test runtimes
-        --allowed-missing            Allowed percentage of missing runtimes (default = 50)
+        --allowed-missing [INT]      Allowed percentage of missing runtimes (default = 50)
         --unknown-runtime [FLOAT]    Use given number as unknown runtime (otherwise use average time)
+        --first-is-1                 Use "1" as TEST_ENV_NUMBER to not reuse the default test environment
+        --fail-fast                  Stop all groups when one group fails (best used with --test-options '--fail-fast' if supported
         --verbose                    Print debug output
-        --verbose-process-command    Print the command that will be executed by each process before it begins
-        --verbose-rerun-command      After a process fails, print the command executed by that process
-        --quiet                      Print only test output
+        --verbose-process-command    Displays only the command that will be executed by each process
+        --verbose-rerun-command      When there are failures, displays the command executed by each process that failed
+        --quiet                      Print only tests output
     -v, --version                    Show Version
     -h, --help                       Show this.
 

lib/parallel_tests/cli.rb

@@ -42,10 +42,10 @@ def execute_in_parallel(items, num_processes, options)
       Tempfile.open 'parallel_tests-lock' do |lock|
         ParallelTests.with_pid_file do
           simulate_output_for_ci options[:serialize_stdout] do
-            Parallel.map(items, :in_threads => num_processes) do |item|
+            Parallel.map(items, in_threads: num_processes) do |item|
               result = yield(item)
               reprint_output(result, lock.path) if options[:serialize_stdout]
-              ParallelTests.stop_all_processes if result[:exit_status] != 0 && options[:fail_fast]
+              ParallelTests.stop_all_processes if options[:fail_fast] && result[:exit_status] != 0
               result
             end
           end
@@ -221,7 +221,7 @@ def parse_options!(argv)
         opts.on("--allowed-missing [INT]", Integer, "Allowed percentage of missing runtimes (default = 50)") { |percent| options[:allowed_missing_percent] = percent }
         opts.on("--unknown-runtime [FLOAT]", Float, "Use given number as unknown runtime (otherwise use average time)") { |time| options[:unknown_runtime] = time }
         opts.on("--first-is-1", "Use \"1\" as TEST_ENV_NUMBER to not reuse the default test environment") { options[:first_is_1] = true }
-        opts.on("--fail-fast", "Stop running the test suite on the first failed test") { options[:fail_fast] = true }
+        opts.on("--fail-fast", "Stop all groups when one group fails (best used with --test-options '--fail-fast' if supported") { options[:fail_fast] = true }
         opts.on("--verbose", "Print debug output") { options[:verbose] = true }
         opts.on("--verbose-process-command", "Displays only the command that will be executed by each process") { options[:verbose_process_command] = true }
         opts.on("--verbose-rerun-command", "When there are failures, displays the command executed by each process that failed") { options[:verbose_rerun_command] = true }

spec/integration_spec.rb

@@ -83,65 +83,54 @@ def self.it_fails_without_any_files(type)
     expect(result).to include '2 processes for 2 specs, ~ 1 specs per process'
   end
 
-  it "fast fail in parallel (enabled)" do
-    # add extra specs to verify they won't be executed
-    # Fail the suite at the first step, and add sleep so the tests were less flaky
-    write 'spec/xxx1_spec.rb', 'describe("it"){it("should"){sleep 1; expect(1).to eq(2)}}'
-    write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}'
-    write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}'
-    write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}'
-    write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}'
-    write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}'
-    # Use 2 processes so it was possible to check that all threads stop
-    # Use --fail-fast option for parallel tests and pass the same option to the rspec
-    # Use group-by found so the order of the executed specs was the same from test to test
-    result = run_tests "spec",
-                       fail: true,
-                       type: 'rspec',
-                       processes: 2,
-                       add: "--group-by found --fail-fast --test-options '--fail-fast'"
+  describe "--fail-fast" do
+    def run_tests(test_option: nil)
+      super(
+        "spec",
+        fail: true,
+        type: 'rspec',
+        processes: 2,
+        # group-by + order for stable execution ... doc and verbose to ease debugging
+        add: "--group-by found --verbose --fail-fast --test-options '--format doc --order defined #{test_option}'"
+      )
+    end
 
-    # test ran and gave their puts
-    expect(result).to include_exactly_times('TESTS', 2)
+    before do
+      write 'spec/xxx1_spec.rb', 'describe("T1"){it("E1"){puts "YE" + "S"; sleep 0.5; expect(1).to eq(2)}}' # group 1 executed
+      write 'spec/xxx2_spec.rb', 'describe("T2"){it("E2"){sleep 1; puts "OK"}}'  # group 2 executed
+      write 'spec/xxx3_spec.rb', 'describe("T3"){it("E3"){puts "NO3"}}' # group 1 skipped
+      write 'spec/xxx4_spec.rb', 'describe("T4"){it("E4"){puts "NO4"}}' # group 2 skipped
+      write 'spec/xxx5_spec.rb', 'describe("T5"){it("E5"){puts "NO5"}}' # group 1 skipped
+      write 'spec/xxx6_spec.rb', 'describe("T6"){it("E6"){puts "NO6"}}' # group 2 skipped
+    end
 
-    # all results present
-    expect(result).to include_exactly_times('1 example, 1 failure', 1) # results
-    expect(result).to include_exactly_times('2 examples, 0 failure', 1) # results
-    expect(result).to include_exactly_times('3 examples, 1 failure', 1) # 1 summary, verify only 3 specs were executed
-    expect(result).to include_exactly_times(/Finished in \d+(\.\d+)? seconds/, 2)
-    expect(result).to include_exactly_times(/Took \d+ seconds/, 1) # parallel summary
+    it "can fail fast on a single test" do
+      result = run_tests(test_option: "--fail-fast")
 
-    # verify that successful run would have 6 specs
-    expect(result).to include '2 processes for 6 specs, ~ 3 specs per process'
-  end
+      expect(result).to include_exactly_times("YES", 1)
+      expect(result).to include_exactly_times("OK", 1) # is allowed to finish but no new test is started after
+      expect(result).to_not include("NO")
 
-  it "fast fail in parallel (disabled)" do
-    write 'spec/xxx1_spec.rb', 'describe("it"){it("should"){expect(1).to eq(2)}}'
-    write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){puts "TEST2"}}'
-    write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){puts "TEST3"}}'
-    write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){puts "TEST4"}}'
-    write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){puts "TEST5"}}'
-    write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){puts "TEST6"}}'
+      expect(result).to include_exactly_times('1 example, 1 failure', 1) # rspec group 1
+      expect(result).to include_exactly_times('1 example, 0 failure', 1) # rspec group 2
+      expect(result).to include_exactly_times('2 examples, 1 failure', 1) # parallel_rspec summary
+
+      expect(result).to include '2 processes for 6 specs, ~ 3 specs per process'
+    end
 
-    result = run_tests "spec",
-                       fail: true,
-                       type: 'rspec',
-                       processes: 2,
-                       add: "--group-by found"
+    it "can fail fast on a single group" do
+      result = run_tests
 
-    # test ran and gave their puts
-    expect(result).to include('TEST2')
-    expect(result).to include('TEST3')
-    expect(result).to include('TEST4')
-    expect(result).to include('TEST5')
-    expect(result).to include('TEST6')
+      expect(result).to include_exactly_times("YES", 1)
+      expect(result).to include_exactly_times("OK", 1) # is allowed to finish but no new test is started after
+      expect(result).to include_exactly_times("NO", 2)
 
-    # all results present
-    expect(result).to include_exactly_times('3 examples, 1 failure', 1) # results
-    expect(result).to include_exactly_times('3 examples, 0 failure', 1) # results
-    expect(result).to include_exactly_times('6 examples, 1 failure', 1) # 1 summary, verify all specs were executed
-    expect(result).to include_exactly_times(/Finished in \d+(\.\d+)? seconds/, 2)
-    expect(result).to include_exactly_times(/Took \d+ seconds/, 1) # parallel summary
+      expect(result).to include_exactly_times('3 examples, 1 failure', 1) # rspec group 1
+      expect(result).to include_exactly_times('1 example, 0 failure', 1) # rspec group 2
+      expect(result).to include_exactly_times('4 examples, 1 failure', 1) # parallel_rspec summary
+
+      expect(result).to include '2 processes for 6 specs, ~ 3 specs per process'
+    end
   end
 
   it "runs tests which outputs accented characters" do
@@ -160,11 +149,11 @@ def test_unicode
         end
       end
     EOF
+
     # Need to tell Ruby to default to utf-8 to simulate environments where
     # this is set. (Otherwise, it defaults to nil and the undefined conversion
     # issue doesn't come up.)
-    result = run_tests('test', fail: true,
-                       export: {'RUBYOPT' => 'Eutf-8:utf-8'})
+    result = run_tests('test', fail: true, export: {'RUBYOPT' => 'Eutf-8:utf-8'})
     expect(result).to include('¯\_(ツ)_/¯')
   end
 
@@ -346,11 +335,11 @@ def test_unicode
   it "runs with PARALLEL_TEST_PROCESSORS processes" do
     skip if RUBY_PLATFORM == "java" # execution expired issue on JRuby
     processes = 5
-    processes.times { |i|
+    processes.times do |i|
       write "spec/x#{i}_spec.rb", "puts %{ENV-\#{ENV['TEST_ENV_NUMBER']}-}"
-    }
+    end
     result = run_tests(
-        "spec", export: {"PARALLEL_TEST_PROCESSORS" => processes.to_s}, processes: processes, type: 'rspec'
+      "spec", export: {"PARALLEL_TEST_PROCESSORS" => processes.to_s}, processes: processes, type: 'rspec'
     )
     expect(result.scan(/ENV-.?-/)).to match_array(["ENV--", "ENV-2-", "ENV-3-", "ENV-4-", "ENV-5-"])
   end