From 28c67479f4f7949aefa0e0eb3998fd98e48e019b Mon Sep 17 00:00:00 2001 From: artem Date: Tue, 7 Jul 2020 15:19:30 +0300 Subject: [PATCH 1/6] Added option which stops all threads when one of them return non zero exit code. So it makes possible to stop whole suite for all threads if every thread will works option `--fail-fast`. --- lib/parallel_tests/cli.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/parallel_tests/cli.rb b/lib/parallel_tests/cli.rb index 66376c7d..98e8166c 100644 --- a/lib/parallel_tests/cli.rb +++ b/lib/parallel_tests/cli.rb @@ -44,6 +44,7 @@ def execute_in_parallel(items, num_processes, options) simulate_output_for_ci options[:serialize_stdout] do Parallel.map(items, :in_threads => num_processes) do |item| result = yield(item) + ParallelTests.stop_all_processes if result[:exit_status] != 0 && options[:fail_fast] reprint_output(result, lock.path) if options[:serialize_stdout] result end @@ -220,6 +221,7 @@ def parse_options!(argv) opts.on("--allowed-missing [INT]", Integer, "Allowed percentage of missing runtimes (default = 50)") { |percent| options[:allowed_missing_percent] = percent } opts.on("--unknown-runtime [FLOAT]", Float, "Use given number as unknown runtime (otherwise use average time)") { |time| options[:unknown_runtime] = time } opts.on("--first-is-1", "Use \"1\" as TEST_ENV_NUMBER to not reuse the default test environment") { options[:first_is_1] = true } + opts.on("--fail-fast", "Stop running the test suite on the first failed test") { options[:fail_fast] = true } opts.on("--verbose", "Print debug output") { options[:verbose] = true } opts.on("--verbose-process-command", "Displays only the command that will be executed by each process") { options[:verbose_process_command] = true } opts.on("--verbose-rerun-command", "When there are failures, displays the command executed by each process that failed") { options[:verbose_rerun_command] = true } From 33658fc6b638e3978616831d3ab0f83d368c4299 Mon Sep 17 00:00:00 2001 From: artem Date: Tue, 7 Jul 2020 17:34:32 +0300 Subject: [PATCH 2/6] Added test to the to cover the feature --- spec/integration_spec.rb | 113 ++++++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 42 deletions(-) diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 7b157bcd..271b3176 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -28,7 +28,7 @@ def bin_folder "#{File.expand_path(File.dirname(__FILE__))}/../bin" end - def executable(options={}) + def executable(options = {}) "ruby #{bin_folder}/parallel_#{options[:type] || 'test'}" end @@ -36,9 +36,9 @@ def ensure_folder(folder) FileUtils.mkpath(folder) unless File.exist?(folder) end - def run_tests(test_folder, options={}) + def run_tests(test_folder, options = {}) ensure_folder folder - processes = "-n #{options[:processes]||2}" unless options[:processes] == false + processes = "-n #{options[:processes] || 2}" unless options[:processes] == false command = "#{executable(options)} #{test_folder} #{processes} #{options[:add]}" result = '' Dir.chdir(folder) do @@ -67,7 +67,7 @@ def self.it_fails_without_any_files(type) write 'spec/xxx_spec.rb', 'describe("it"){it("should"){puts "TEST1"}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){puts "TEST2"}}' # set processes to false so we verify empty groups are discarded by default - result = run_tests "spec", :type => 'rspec', :processes => 4 + result = run_tests "spec", type: 'rspec', processes: 4 # test ran and gave their puts expect(result).to include('TEST1') @@ -83,9 +83,38 @@ def self.it_fails_without_any_files(type) expect(result).to include '2 processes for 2 specs, ~ 1 specs per process' end + it "fast fail in parallel" do + write 'spec/xxx1_spec.rb', 'describe("it"){it("should"){sleep 1; expect(1).to eq(2)}}' + write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST2"}}' + write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST3"}}' + write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST4"}}' + write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST5"}}' + write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST6"}}' + # set processes to false so we verify empty groups are discarded by default + result = run_tests "spec", + fail: true, + type: 'rspec', + processes: 2, + add: "--group-by found --fail-fast --test-options '--fail-fast'" + + # test ran and gave their puts + expect(result).to include('TEST2') + expect(result).to include('TEST4') + + # all results present + expect(result).to include_exactly_times('1 example, 1 failure', 1) # results + expect(result).to include_exactly_times('2 examples, 0 failure', 1) # results + expect(result).to include_exactly_times('3 examples, 1 failure', 1) # 1 summary + expect(result).to include_exactly_times(/Finished in \d+(\.\d+)? seconds/, 2) + expect(result).to include_exactly_times(/Took \d+ seconds/, 1) # parallel summary + + # verify empty groups are discarded. if retained then it'd say 4 processes for 2 specs + expect(result).to include '2 processes for 6 specs, ~ 3 specs per process' + end + it "runs tests which outputs accented characters" do write "spec/xxx_spec.rb", "#encoding: utf-8\ndescribe('it'){it('should'){puts 'Byłem tu'}}" - result = run_tests "spec", :type => 'rspec' + result = run_tests "spec", type: 'rspec' # test ran and gave their puts expect(result).to include('Byłem tu') end @@ -102,14 +131,14 @@ def test_unicode # Need to tell Ruby to default to utf-8 to simulate environments where # this is set. (Otherwise, it defaults to nil and the undefined conversion # issue doesn't come up.) - result = run_tests('test', :fail => true, - :export => {'RUBYOPT' => 'Eutf-8:utf-8'}) + result = run_tests('test', fail: true, + export: {'RUBYOPT' => 'Eutf-8:utf-8'}) expect(result).to include('¯\_(ツ)_/¯') end it "does not run any tests if there are none" do write 'spec/xxx_spec.rb', '1' - result = run_tests "spec", :type => 'rspec' + result = run_tests "spec", type: 'rspec' expect(result).to include('No examples found') expect(result).to include('Took') end @@ -117,7 +146,7 @@ def test_unicode it "shows command and rerun with --verbose" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){puts "TEST1"}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){expect(1).to eq(2)}}' - result = run_tests "spec --verbose", :type => 'rspec', :fail => true + result = run_tests "spec --verbose", type: 'rspec', fail: true expect(result).to include printed_commands expect(result).to include printed_rerun expect(result).to include "bundle exec rspec spec/xxx_spec.rb" @@ -126,14 +155,14 @@ def test_unicode it "shows only rerun with --verbose-rerun-command" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){expect(1).to eq(2)}}' - result = run_tests "spec --verbose-rerun-command", :type => 'rspec', :fail => true + result = run_tests "spec --verbose-rerun-command", type: 'rspec', fail: true expect(result).to include printed_rerun expect(result).to_not include printed_commands end it "shows only process with --verbose-process-command" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){expect(1).to eq(2)}}' - result = run_tests "spec --verbose-process-command", :type => 'rspec', :fail => true + result = run_tests "spec --verbose-process-command", type: 'rspec', fail: true expect(result).to_not include printed_rerun expect(result).to include printed_commands end @@ -141,7 +170,7 @@ def test_unicode it "fails when tests fail" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){puts "TEST1"}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){expect(1).to eq(2)}}' - result = run_tests "spec", :fail => true, :type => 'rspec' + result = run_tests "spec", fail: true, type: 'rspec' expect(result).to include_exactly_times('1 example, 1 failure', 1) expect(result).to include_exactly_times('1 example, 0 failure', 1) @@ -151,7 +180,7 @@ def test_unicode it "can serialize stdout" do write 'spec/xxx_spec.rb', '5.times{describe("it"){it("should"){sleep 0.01; puts "TEST1"}}}' write 'spec/xxx2_spec.rb', 'sleep 0.01; 5.times{describe("it"){it("should"){sleep 0.01; puts "TEST2"}}}' - result = run_tests "spec", :type => 'rspec', :add => "--serialize-stdout" + result = run_tests "spec", type: 'rspec', add: "--serialize-stdout" expect(result).not_to match(/TEST1.*TEST2.*TEST1/m) expect(result).not_to match(/TEST2.*TEST1.*TEST2/m) @@ -160,20 +189,20 @@ def test_unicode it "can show simulated output when serializing stdout" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){sleep 0.5; puts "TEST1"}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST2"}}' - result = run_tests "spec", :type => 'rspec', :add => "--serialize-stdout", export: {'PARALLEL_TEST_HEARTBEAT_INTERVAL' => '0.01'} + result = run_tests "spec", type: 'rspec', add: "--serialize-stdout", export: {'PARALLEL_TEST_HEARTBEAT_INTERVAL' => '0.01'} expect(result).to match(/\.{4}.*TEST1.*\.{4}.*TEST2/m) end it "can show simulated output preceded by command when serializing stdout with verbose option" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST1"}}' - result = run_tests "spec --verbose", :type => 'rspec', :add => "--serialize-stdout", export: {'PARALLEL_TEST_HEARTBEAT_INTERVAL' => '0.02'} + result = run_tests "spec --verbose", type: 'rspec', add: "--serialize-stdout", export: {'PARALLEL_TEST_HEARTBEAT_INTERVAL' => '0.02'} expect(result).to match(/\.{5}.*\nbundle exec rspec spec\/xxx_spec\.rb\nTEST1/m) end it "can serialize stdout and stderr" do write 'spec/xxx_spec.rb', '5.times{describe("it"){it("should"){sleep 0.01; $stderr.puts "errTEST1"; puts "TEST1"}}}' write 'spec/xxx2_spec.rb', 'sleep 0.01; 5.times{describe("it"){it("should"){sleep 0.01; $stderr.puts "errTEST2"; puts "TEST2"}}}' - result = run_tests "spec", :type => 'rspec', :add => "--serialize-stdout --combine-stderr" + result = run_tests "spec", type: 'rspec', add: "--serialize-stdout --combine-stderr" expect(result).not_to match(/TEST1.*TEST2.*TEST1/m) expect(result).not_to match(/TEST2.*TEST1.*TEST2/m) @@ -228,7 +257,7 @@ def test_unicode it "runs with --group-by found" do # it only tests that it does not blow up, as it did before fixing... write "spec/x1_spec.rb", "puts 'TEST111'" - run_tests "spec", :type => 'rspec', :add => '--group-by found' + run_tests "spec", type: 'rspec', add: '--group-by found' end it "runs in parallel" do @@ -269,7 +298,7 @@ def test_unicode write "spec/x1_spec.rb", "puts 'TEST111'" write "spec/x2_spec.rb", "puts 'TEST222'" write "spec/x3_spec.rb", "puts 'TEST333'" - result = run_tests "spec/x1_spec.rb spec/x3_spec.rb", :type => 'rspec' + result = run_tests "spec/x1_spec.rb spec/x3_spec.rb", type: 'rspec' expect(result).to include('TEST111') expect(result).to include('TEST333') expect(result).not_to include('TEST222') @@ -289,7 +318,7 @@ def test_unicode write "spec/x#{i}_spec.rb", "puts %{ENV-\#{ENV['TEST_ENV_NUMBER']}-}" } result = run_tests( - "spec", export: {"PARALLEL_TEST_PROCESSORS" => processes.to_s}, processes: processes, type: 'rspec' + "spec", export: {"PARALLEL_TEST_PROCESSORS" => processes.to_s}, processes: processes, type: 'rspec' ) expect(result.scan(/ENV-.?-/)).to match_array(["ENV--", "ENV-2-", "ENV-3-", "ENV-4-", "ENV-5-"]) end @@ -298,7 +327,7 @@ def test_unicode write "spec/x_spec.rb", "puts 'TESTXXX'" write "spec/y_spec.rb", "puts 'TESTYYY'" write "spec/z_spec.rb", "puts 'TESTZZZ'" - result = run_tests "spec", :add => "-p '^spec/(x|z)'", :type => "rspec" + result = run_tests "spec", add: "-p '^spec/(x|z)'", type: "rspec" expect(result).to include('TESTXXX') expect(result).not_to include('TESTYYY') expect(result).to include('TESTZZZ') @@ -308,7 +337,7 @@ def test_unicode write "spec/x_spec.rb", "puts 'TESTXXX'" write "spec/acceptance/y_spec.rb", "puts 'TESTYYY'" write "spec/integration/z_spec.rb", "puts 'TESTZZZ'" - result = run_tests "spec", :add => "--exclude-pattern 'spec/(integration|acceptance)'", :type => "rspec" + result = run_tests "spec", add: "--exclude-pattern 'spec/(integration|acceptance)'", type: "rspec" expect(result).to include('TESTXXX') expect(result).not_to include('TESTYYY') expect(result).not_to include('TESTZZZ') @@ -320,7 +349,7 @@ def test_unicode write "test/b_test.rb", "sleep 1; puts 'OutputB'" write "test/c_test.rb", "sleep 1.5; puts 'OutputC'" write "test/d_test.rb", "sleep 2; puts 'OutputD'" - actual = run_tests("test", :processes => 4).scan(/Output[ABCD]/) + actual = run_tests("test", processes: 4).scan(/Output[ABCD]/) actual_sorted = [*actual[0..2].sort, actual[3]] expect(actual_sorted).to match(["OutputB", "OutputC", "OutputD", "OutputA"]) end @@ -330,11 +359,11 @@ def test_unicode write "test/long_test.rb", "puts 'this is a long test'" write "test/short_test.rb", "puts 'short test'" - group_1_result = run_tests("test", :processes => 2, :add => '--only-group 1') + group_1_result = run_tests("test", processes: 2, add: '--only-group 1') expect(group_1_result).to include("this is a long test") expect(group_1_result).not_to include("short test") - group_2_result = run_tests("test", :processes => 2, :add => '--only-group 2') + group_2_result = run_tests("test", processes: 2, add: '--only-group 2') expect(group_2_result).not_to include("this is a long test") expect(group_2_result).to include("short test") end @@ -345,7 +374,7 @@ def test_unicode it "captures seed with random failures with --verbose" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){puts "TEST1"}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){1.should == 2}}' - result = run_tests "spec --verbose", :add => "--test-options '--seed 1234'", :fail => true, :type => 'rspec' + result = run_tests "spec --verbose", add: "--test-options '--seed 1234'", fail: true, type: 'rspec' expect(result).to include("Randomized with seed 1234") expect(result).to include("bundle exec rspec spec/xxx2_spec.rb --seed 1234") end @@ -360,7 +389,7 @@ def test_unicode it "passes test options" do write "test/x1_test.rb", "require 'test/unit'; class XTest < Test::Unit::TestCase; def test_xxx; end; end" - result = run_tests("test", :add => '--test-options "-v"') + result = run_tests("test", add: '--test-options "-v"') expect(result).to include('test_xxx') # verbose output of every test end @@ -380,7 +409,7 @@ def test_unicode it "runs tests which outputs accented characters" do write "features/good1.feature", "Feature: xxx\n Scenario: xxx\n Given I print accented characters" write "features/steps/a.rb", "#encoding: utf-8\nGiven('I print accented characters'){ puts \"I tu też\" }" - result = run_tests "features", :type => "cucumber", :add => '--pattern good' + result = run_tests "features", type: "cucumber", add: '--pattern good' expect(result).to include('I tu też') end @@ -390,7 +419,7 @@ def test_unicode write "features/b.feature", "Feature: xxx\n Scenario: xxx\n Given I FAIL" write "features/steps/a.rb", "Given('I print TEST_ENV_NUMBER'){ puts \"YOUR TEST ENV IS \#{ENV['TEST_ENV_NUMBER']}!\" }" - result = run_tests "features", :type => "cucumber", :add => '--pattern good' + result = run_tests "features", type: "cucumber", add: '--pattern good' expect(result).to include('YOUR TEST ENV IS 2!') expect(result).to include('YOUR TEST ENV IS !') @@ -406,7 +435,7 @@ def test_unicode # needs sleep so that runtime loggers dont overwrite each other initially write "features/good#{i}.feature", "Feature: xxx\n Scenario: xxx\n Given I print TEST_ENV_NUMBER\n And I sleep a bit" } - run_tests "features", :type => "cucumber" + run_tests "features", type: "cucumber" expect(read(log).gsub(/\.\d+/, '').split("\n")).to match_array(["features/good0.feature:0", "features/good1.feature:0"]) end @@ -414,7 +443,7 @@ def test_unicode 2.times { |i| write "features/good#{i}.feature", "Feature: xxx\n Scenario: xxx\n Given I print TEST_ENV_NUMBER" } - result = run_tests "features", :type => "cucumber", :add => '-n 3' + result = run_tests "features", type: "cucumber", add: '-n 3' expect(result.scan(/YOUR TEST ENV IS \d?!/).sort).to eq(["YOUR TEST ENV IS !", "YOUR TEST ENV IS 2!"]) end @@ -424,13 +453,13 @@ def test_unicode write "features/pass.feature", "Feature: xxx\n Scenario: xxx\n Given I pass" write "features/fail1.feature", "Feature: xxx\n Scenario: xxx\n Given I fail" write "features/fail2.feature", "Feature: xxx\n Scenario: xxx\n Given I fail" - results = run_tests "features", :processes => 3, :type => "cucumber", :fail => true + results = run_tests "features", processes: 3, type: "cucumber", fail: true failing_scenarios = if Gem.win_platform? - ["cucumber features/fail1.feature:2 # Scenario: xxx", "cucumber features/fail2.feature:2 # Scenario: xxx"] - else - ["cucumber features/fail2.feature:2 # Scenario: xxx", "cucumber features/fail1.feature:2 # Scenario: xxx"] - end + ["cucumber features/fail1.feature:2 # Scenario: xxx", "cucumber features/fail2.feature:2 # Scenario: xxx"] + else + ["cucumber features/fail2.feature:2 # Scenario: xxx", "cucumber features/fail1.feature:2 # Scenario: xxx"] + end expect(results).to include <<-EOF.gsub(' ', '') Failing Scenarios: @@ -459,7 +488,7 @@ def test_unicode | one | | two | EOS - result = run_tests "features", :type => "cucumber", :add => "--group-by scenarios" + result = run_tests "features", type: "cucumber", add: "--group-by scenarios" expect(result).to include("2 processes for 4 scenarios") end @@ -467,14 +496,14 @@ def test_unicode write "features/good1.feature", "Feature: xxx\n Scenario: xxx\n Given I print TEST_ENV_NUMBER" write "features/good2.feature", "Feature: xxx\n Scenario: xxx\n Given I print TEST_ENV_NUMBER" - result = run_tests "features", :type => "cucumber", :add => '--group-by steps' + result = run_tests "features", type: "cucumber", add: '--group-by steps' expect(result).to include("2 processes for 2 features") end it "captures seed with random failures with --verbose" do write "features/good1.feature", "Feature: xxx\n Scenario: xxx\n Given I fail" - result = run_tests "features --verbose", :type => "cucumber", :add => '--test-options "--order random:1234"', :fail => true + result = run_tests "features --verbose", type: "cucumber", add: '--test-options "--order random:1234"', fail: true expect(result).to include("Randomized with seed 1234") expect(result).to match(%r{bundle exec cucumber "?features/good1.feature"? --order random:1234}) end @@ -497,7 +526,7 @@ class A < Spinach::FeatureSteps it "runs tests which outputs accented characters" do write "features/good1.feature", "Feature: a\n Scenario: xxx\n Given I print accented characters" write "features/steps/a.rb", "#encoding: utf-8\nclass A < Spinach::FeatureSteps\nGiven 'I print accented characters' do\n puts \"I tu też\" \n end\nend" - result = run_tests "features", :type => "spinach", :add => 'features/good1.feature' #, :add => '--pattern good' + result = run_tests "features", type: "spinach", add: 'features/good1.feature' #, :add => '--pattern good' expect(result).to include('I tu też') end @@ -506,7 +535,7 @@ class A < Spinach::FeatureSteps write "features/good2.feature", "Feature: a\n Scenario: xxx\n Given I print TEST_ENV_NUMBER" write "features/b.feature", "Feature: b\n Scenario: xxx\n Given I FAIL" #Expect this not to be run - result = run_tests "features", :type => "spinach", :add => '--pattern good' + result = run_tests "features", type: "spinach", add: '--pattern good' expect(result).to include('YOUR TEST ENV IS 2!') expect(result).to include('YOUR TEST ENV IS !') @@ -522,7 +551,7 @@ class A < Spinach::FeatureSteps # needs sleep so that runtime loggers dont overwrite each other initially write "features/good#{i}.feature", "Feature: A\n Scenario: xxx\n Given I print TEST_ENV_NUMBER\n And I sleep a bit" } - run_tests "features", :type => "spinach" + run_tests "features", type: "spinach" expect(read(log).gsub(/\.\d+/, '').split("\n")).to match_array(["features/good0.feature:0", "features/good1.feature:0"]) end @@ -530,7 +559,7 @@ class A < Spinach::FeatureSteps 2.times { |i| write "features/good#{i}.feature", "Feature: A\n Scenario: xxx\n Given I print TEST_ENV_NUMBER\n" } - result = run_tests "features", :type => "spinach", :add => '-n 3' + result = run_tests "features", type: "spinach", add: '-n 3' expect(result.scan(/YOUR TEST ENV IS \d?!/).sort).to eq(["YOUR TEST ENV IS !", "YOUR TEST ENV IS 2!"]) end From a91f7519605c4953b799185d448cc2868ad9ad8b Mon Sep 17 00:00:00 2001 From: artem Date: Tue, 7 Jul 2020 21:49:37 +0300 Subject: [PATCH 3/6] Updated test, added case check without --fail-fast --- spec/integration_spec.rb | 43 +++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 271b3176..30a2669f 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -83,14 +83,18 @@ def self.it_fails_without_any_files(type) expect(result).to include '2 processes for 2 specs, ~ 1 specs per process' end - it "fast fail in parallel" do + it "fast fail in parallel (enabled)" do + # add extra specs to verify they won't be executed + # Fail the suite at the first step, and add sleep so the tests were less flaky write 'spec/xxx1_spec.rb', 'describe("it"){it("should"){sleep 1; expect(1).to eq(2)}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST2"}}' write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST3"}}' write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST4"}}' write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST5"}}' write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST6"}}' - # set processes to false so we verify empty groups are discarded by default + # Use 2 processes so it was possible to check that all threads stop + # Use --fail-fast option for parallel tests and pass the same option to the rspec + # Use group-by found so the order of the executed specs was the same from test to test result = run_tests "spec", fail: true, type: 'rspec', @@ -98,20 +102,49 @@ def self.it_fails_without_any_files(type) add: "--group-by found --fail-fast --test-options '--fail-fast'" # test ran and gave their puts - expect(result).to include('TEST2') expect(result).to include('TEST4') + expect(result).to include('TEST5') # all results present expect(result).to include_exactly_times('1 example, 1 failure', 1) # results expect(result).to include_exactly_times('2 examples, 0 failure', 1) # results - expect(result).to include_exactly_times('3 examples, 1 failure', 1) # 1 summary + expect(result).to include_exactly_times('3 examples, 1 failure', 1) # 1 summary, verify only 3 specs were executed expect(result).to include_exactly_times(/Finished in \d+(\.\d+)? seconds/, 2) expect(result).to include_exactly_times(/Took \d+ seconds/, 1) # parallel summary - # verify empty groups are discarded. if retained then it'd say 4 processes for 2 specs + # verify that successful run would have 6 specs expect(result).to include '2 processes for 6 specs, ~ 3 specs per process' end + it "fast fail in parallel (disabled)" do + write 'spec/xxx1_spec.rb', 'describe("it"){it("should"){expect(1).to eq(2)}}' + write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){puts "TEST2"}}' + write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){puts "TEST3"}}' + write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){puts "TEST4"}}' + write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){puts "TEST5"}}' + write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){puts "TEST6"}}' + + result = run_tests "spec", + fail: true, + type: 'rspec', + processes: 2, + add: "--group-by found" + + # test ran and gave their puts + expect(result).to include('TEST2') + expect(result).to include('TEST3') + expect(result).to include('TEST4') + expect(result).to include('TEST5') + expect(result).to include('TEST6') + + # all results present + expect(result).to include_exactly_times('3 examples, 1 failure', 1) # results + expect(result).to include_exactly_times('3 examples, 0 failure', 1) # results + expect(result).to include_exactly_times('6 examples, 1 failure', 1) # 1 summary, verify all specs were executed + expect(result).to include_exactly_times(/Finished in \d+(\.\d+)? seconds/, 2) + expect(result).to include_exactly_times(/Took \d+ seconds/, 1) # parallel summary + end + it "runs tests which outputs accented characters" do write "spec/xxx_spec.rb", "#encoding: utf-8\ndescribe('it'){it('should'){puts 'Byłem tu'}}" result = run_tests "spec", type: 'rspec' From 8169807654e61c08922e4ad983f174429b1b449a Mon Sep 17 00:00:00 2001 From: artem Date: Tue, 7 Jul 2020 21:58:41 +0300 Subject: [PATCH 4/6] Updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db05cc30..7dfc3dee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ ### Added -- None +- `--fail-fast` options which stops all threads if one of them return not zero exit code. Which add possibility to stop whole suite if one test failed. Works if the option `--fail-fast` enabled for the rspec (passed to the test_options: `--test-options '--fail-fast'` or enabled at the .rspec_parallel file). ### Fixed From e81e6dcbc3b0ef4b6f5b11af0320e347e453e883 Mon Sep 17 00:00:00 2001 From: artem Date: Tue, 7 Jul 2020 22:46:55 +0300 Subject: [PATCH 5/6] Different platform separate files in to different group. So different specs will be executed and different puts will be displayed. Changed puts to be the same at all specs to assert amount of puts outputs which should be the same --- spec/integration_spec.rb | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 30a2669f..8a4c05f0 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -87,11 +87,11 @@ def self.it_fails_without_any_files(type) # add extra specs to verify they won't be executed # Fail the suite at the first step, and add sleep so the tests were less flaky write 'spec/xxx1_spec.rb', 'describe("it"){it("should"){sleep 1; expect(1).to eq(2)}}' - write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST2"}}' - write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST3"}}' - write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST4"}}' - write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST5"}}' - write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST6"}}' + write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' + write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' + write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' + write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' + write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' # Use 2 processes so it was possible to check that all threads stop # Use --fail-fast option for parallel tests and pass the same option to the rspec # Use group-by found so the order of the executed specs was the same from test to test @@ -102,8 +102,7 @@ def self.it_fails_without_any_files(type) add: "--group-by found --fail-fast --test-options '--fail-fast'" # test ran and gave their puts - expect(result).to include('TEST4') - expect(result).to include('TEST5') + expect(result).to include_exactly_times('TESTS', 2) # all results present expect(result).to include_exactly_times('1 example, 1 failure', 1) # results From e1f35ebb64de9b80b12cac3dca48b88743695cad Mon Sep 17 00:00:00 2001 From: artem Date: Tue, 14 Jul 2020 00:17:10 +0300 Subject: [PATCH 6/6] Updated indentation as was before Changed order of the command, reprint output before stopping all the processes --- lib/parallel_tests/cli.rb | 2 +- spec/integration_spec.rb | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/parallel_tests/cli.rb b/lib/parallel_tests/cli.rb index 98e8166c..4d276f71 100644 --- a/lib/parallel_tests/cli.rb +++ b/lib/parallel_tests/cli.rb @@ -44,8 +44,8 @@ def execute_in_parallel(items, num_processes, options) simulate_output_for_ci options[:serialize_stdout] do Parallel.map(items, :in_threads => num_processes) do |item| result = yield(item) - ParallelTests.stop_all_processes if result[:exit_status] != 0 && options[:fail_fast] reprint_output(result, lock.path) if options[:serialize_stdout] + ParallelTests.stop_all_processes if result[:exit_status] != 0 && options[:fail_fast] result end end diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 8a4c05f0..f43b3176 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -488,10 +488,10 @@ def test_unicode results = run_tests "features", processes: 3, type: "cucumber", fail: true failing_scenarios = if Gem.win_platform? - ["cucumber features/fail1.feature:2 # Scenario: xxx", "cucumber features/fail2.feature:2 # Scenario: xxx"] - else - ["cucumber features/fail2.feature:2 # Scenario: xxx", "cucumber features/fail1.feature:2 # Scenario: xxx"] - end + ["cucumber features/fail1.feature:2 # Scenario: xxx", "cucumber features/fail2.feature:2 # Scenario: xxx"] + else + ["cucumber features/fail2.feature:2 # Scenario: xxx", "cucumber features/fail1.feature:2 # Scenario: xxx"] + end expect(results).to include <<-EOF.gsub(' ', '') Failing Scenarios: