From 28c67479f4f7949aefa0e0eb3998fd98e48e019b Mon Sep 17 00:00:00 2001 From: artem Date: Tue, 7 Jul 2020 15:19:30 +0300 Subject: [PATCH 1/7] Added option which stops all threads when one of them return non zero exit code. So it makes possible to stop whole suite for all threads if every thread will works option `--fail-fast`. --- lib/parallel_tests/cli.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/parallel_tests/cli.rb b/lib/parallel_tests/cli.rb index 66376c7d..98e8166c 100644 --- a/lib/parallel_tests/cli.rb +++ b/lib/parallel_tests/cli.rb @@ -44,6 +44,7 @@ def execute_in_parallel(items, num_processes, options) simulate_output_for_ci options[:serialize_stdout] do Parallel.map(items, :in_threads => num_processes) do |item| result = yield(item) + ParallelTests.stop_all_processes if result[:exit_status] != 0 && options[:fail_fast] reprint_output(result, lock.path) if options[:serialize_stdout] result end @@ -220,6 +221,7 @@ def parse_options!(argv) opts.on("--allowed-missing [INT]", Integer, "Allowed percentage of missing runtimes (default = 50)") { |percent| options[:allowed_missing_percent] = percent } opts.on("--unknown-runtime [FLOAT]", Float, "Use given number as unknown runtime (otherwise use average time)") { |time| options[:unknown_runtime] = time } opts.on("--first-is-1", "Use \"1\" as TEST_ENV_NUMBER to not reuse the default test environment") { options[:first_is_1] = true } + opts.on("--fail-fast", "Stop running the test suite on the first failed test") { options[:fail_fast] = true } opts.on("--verbose", "Print debug output") { options[:verbose] = true } opts.on("--verbose-process-command", "Displays only the command that will be executed by each process") { options[:verbose_process_command] = true } opts.on("--verbose-rerun-command", "When there are failures, displays the command executed by each process that failed") { options[:verbose_rerun_command] = true } From 33658fc6b638e3978616831d3ab0f83d368c4299 Mon Sep 17 00:00:00 2001 From: artem Date: Tue, 7 Jul 2020 17:34:32 +0300 Subject: [PATCH 2/7] Added test to the to cover the feature --- spec/integration_spec.rb | 113 ++++++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 42 deletions(-) diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 7b157bcd..271b3176 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -28,7 +28,7 @@ def bin_folder "#{File.expand_path(File.dirname(__FILE__))}/../bin" end - def executable(options={}) + def executable(options = {}) "ruby #{bin_folder}/parallel_#{options[:type] || 'test'}" end @@ -36,9 +36,9 @@ def ensure_folder(folder) FileUtils.mkpath(folder) unless File.exist?(folder) end - def run_tests(test_folder, options={}) + def run_tests(test_folder, options = {}) ensure_folder folder - processes = "-n #{options[:processes]||2}" unless options[:processes] == false + processes = "-n #{options[:processes] || 2}" unless options[:processes] == false command = "#{executable(options)} #{test_folder} #{processes} #{options[:add]}" result = '' Dir.chdir(folder) do @@ -67,7 +67,7 @@ def self.it_fails_without_any_files(type) write 'spec/xxx_spec.rb', 'describe("it"){it("should"){puts "TEST1"}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){puts "TEST2"}}' # set processes to false so we verify empty groups are discarded by default - result = run_tests "spec", :type => 'rspec', :processes => 4 + result = run_tests "spec", type: 'rspec', processes: 4 # test ran and gave their puts expect(result).to include('TEST1') @@ -83,9 +83,38 @@ def self.it_fails_without_any_files(type) expect(result).to include '2 processes for 2 specs, ~ 1 specs per process' end + it "fast fail in parallel" do + write 'spec/xxx1_spec.rb', 'describe("it"){it("should"){sleep 1; expect(1).to eq(2)}}' + write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST2"}}' + write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST3"}}' + write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST4"}}' + write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST5"}}' + write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST6"}}' + # set processes to false so we verify empty groups are discarded by default + result = run_tests "spec", + fail: true, + type: 'rspec', + processes: 2, + add: "--group-by found --fail-fast --test-options '--fail-fast'" + + # test ran and gave their puts + expect(result).to include('TEST2') + expect(result).to include('TEST4') + + # all results present + expect(result).to include_exactly_times('1 example, 1 failure', 1) # results + expect(result).to include_exactly_times('2 examples, 0 failure', 1) # results + expect(result).to include_exactly_times('3 examples, 1 failure', 1) # 1 summary + expect(result).to include_exactly_times(/Finished in \d+(\.\d+)? seconds/, 2) + expect(result).to include_exactly_times(/Took \d+ seconds/, 1) # parallel summary + + # verify empty groups are discarded. if retained then it'd say 4 processes for 2 specs + expect(result).to include '2 processes for 6 specs, ~ 3 specs per process' + end + it "runs tests which outputs accented characters" do write "spec/xxx_spec.rb", "#encoding: utf-8\ndescribe('it'){it('should'){puts 'Byłem tu'}}" - result = run_tests "spec", :type => 'rspec' + result = run_tests "spec", type: 'rspec' # test ran and gave their puts expect(result).to include('Byłem tu') end @@ -102,14 +131,14 @@ def test_unicode # Need to tell Ruby to default to utf-8 to simulate environments where # this is set. (Otherwise, it defaults to nil and the undefined conversion # issue doesn't come up.) - result = run_tests('test', :fail => true, - :export => {'RUBYOPT' => 'Eutf-8:utf-8'}) + result = run_tests('test', fail: true, + export: {'RUBYOPT' => 'Eutf-8:utf-8'}) expect(result).to include('¯\_(ツ)_/¯') end it "does not run any tests if there are none" do write 'spec/xxx_spec.rb', '1' - result = run_tests "spec", :type => 'rspec' + result = run_tests "spec", type: 'rspec' expect(result).to include('No examples found') expect(result).to include('Took') end @@ -117,7 +146,7 @@ def test_unicode it "shows command and rerun with --verbose" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){puts "TEST1"}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){expect(1).to eq(2)}}' - result = run_tests "spec --verbose", :type => 'rspec', :fail => true + result = run_tests "spec --verbose", type: 'rspec', fail: true expect(result).to include printed_commands expect(result).to include printed_rerun expect(result).to include "bundle exec rspec spec/xxx_spec.rb" @@ -126,14 +155,14 @@ def test_unicode it "shows only rerun with --verbose-rerun-command" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){expect(1).to eq(2)}}' - result = run_tests "spec --verbose-rerun-command", :type => 'rspec', :fail => true + result = run_tests "spec --verbose-rerun-command", type: 'rspec', fail: true expect(result).to include printed_rerun expect(result).to_not include printed_commands end it "shows only process with --verbose-process-command" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){expect(1).to eq(2)}}' - result = run_tests "spec --verbose-process-command", :type => 'rspec', :fail => true + result = run_tests "spec --verbose-process-command", type: 'rspec', fail: true expect(result).to_not include printed_rerun expect(result).to include printed_commands end @@ -141,7 +170,7 @@ def test_unicode it "fails when tests fail" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){puts "TEST1"}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){expect(1).to eq(2)}}' - result = run_tests "spec", :fail => true, :type => 'rspec' + result = run_tests "spec", fail: true, type: 'rspec' expect(result).to include_exactly_times('1 example, 1 failure', 1) expect(result).to include_exactly_times('1 example, 0 failure', 1) @@ -151,7 +180,7 @@ def test_unicode it "can serialize stdout" do write 'spec/xxx_spec.rb', '5.times{describe("it"){it("should"){sleep 0.01; puts "TEST1"}}}' write 'spec/xxx2_spec.rb', 'sleep 0.01; 5.times{describe("it"){it("should"){sleep 0.01; puts "TEST2"}}}' - result = run_tests "spec", :type => 'rspec', :add => "--serialize-stdout" + result = run_tests "spec", type: 'rspec', add: "--serialize-stdout" expect(result).not_to match(/TEST1.*TEST2.*TEST1/m) expect(result).not_to match(/TEST2.*TEST1.*TEST2/m) @@ -160,20 +189,20 @@ def test_unicode it "can show simulated output when serializing stdout" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){sleep 0.5; puts "TEST1"}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST2"}}' - result = run_tests "spec", :type => 'rspec', :add => "--serialize-stdout", export: {'PARALLEL_TEST_HEARTBEAT_INTERVAL' => '0.01'} + result = run_tests "spec", type: 'rspec', add: "--serialize-stdout", export: {'PARALLEL_TEST_HEARTBEAT_INTERVAL' => '0.01'} expect(result).to match(/\.{4}.*TEST1.*\.{4}.*TEST2/m) end it "can show simulated output preceded by command when serializing stdout with verbose option" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST1"}}' - result = run_tests "spec --verbose", :type => 'rspec', :add => "--serialize-stdout", export: {'PARALLEL_TEST_HEARTBEAT_INTERVAL' => '0.02'} + result = run_tests "spec --verbose", type: 'rspec', add: "--serialize-stdout", export: {'PARALLEL_TEST_HEARTBEAT_INTERVAL' => '0.02'} expect(result).to match(/\.{5}.*\nbundle exec rspec spec\/xxx_spec\.rb\nTEST1/m) end it "can serialize stdout and stderr" do write 'spec/xxx_spec.rb', '5.times{describe("it"){it("should"){sleep 0.01; $stderr.puts "errTEST1"; puts "TEST1"}}}' write 'spec/xxx2_spec.rb', 'sleep 0.01; 5.times{describe("it"){it("should"){sleep 0.01; $stderr.puts "errTEST2"; puts "TEST2"}}}' - result = run_tests "spec", :type => 'rspec', :add => "--serialize-stdout --combine-stderr" + result = run_tests "spec", type: 'rspec', add: "--serialize-stdout --combine-stderr" expect(result).not_to match(/TEST1.*TEST2.*TEST1/m) expect(result).not_to match(/TEST2.*TEST1.*TEST2/m) @@ -228,7 +257,7 @@ def test_unicode it "runs with --group-by found" do # it only tests that it does not blow up, as it did before fixing... write "spec/x1_spec.rb", "puts 'TEST111'" - run_tests "spec", :type => 'rspec', :add => '--group-by found' + run_tests "spec", type: 'rspec', add: '--group-by found' end it "runs in parallel" do @@ -269,7 +298,7 @@ def test_unicode write "spec/x1_spec.rb", "puts 'TEST111'" write "spec/x2_spec.rb", "puts 'TEST222'" write "spec/x3_spec.rb", "puts 'TEST333'" - result = run_tests "spec/x1_spec.rb spec/x3_spec.rb", :type => 'rspec' + result = run_tests "spec/x1_spec.rb spec/x3_spec.rb", type: 'rspec' expect(result).to include('TEST111') expect(result).to include('TEST333') expect(result).not_to include('TEST222') @@ -289,7 +318,7 @@ def test_unicode write "spec/x#{i}_spec.rb", "puts %{ENV-\#{ENV['TEST_ENV_NUMBER']}-}" } result = run_tests( - "spec", export: {"PARALLEL_TEST_PROCESSORS" => processes.to_s}, processes: processes, type: 'rspec' + "spec", export: {"PARALLEL_TEST_PROCESSORS" => processes.to_s}, processes: processes, type: 'rspec' ) expect(result.scan(/ENV-.?-/)).to match_array(["ENV--", "ENV-2-", "ENV-3-", "ENV-4-", "ENV-5-"]) end @@ -298,7 +327,7 @@ def test_unicode write "spec/x_spec.rb", "puts 'TESTXXX'" write "spec/y_spec.rb", "puts 'TESTYYY'" write "spec/z_spec.rb", "puts 'TESTZZZ'" - result = run_tests "spec", :add => "-p '^spec/(x|z)'", :type => "rspec" + result = run_tests "spec", add: "-p '^spec/(x|z)'", type: "rspec" expect(result).to include('TESTXXX') expect(result).not_to include('TESTYYY') expect(result).to include('TESTZZZ') @@ -308,7 +337,7 @@ def test_unicode write "spec/x_spec.rb", "puts 'TESTXXX'" write "spec/acceptance/y_spec.rb", "puts 'TESTYYY'" write "spec/integration/z_spec.rb", "puts 'TESTZZZ'" - result = run_tests "spec", :add => "--exclude-pattern 'spec/(integration|acceptance)'", :type => "rspec" + result = run_tests "spec", add: "--exclude-pattern 'spec/(integration|acceptance)'", type: "rspec" expect(result).to include('TESTXXX') expect(result).not_to include('TESTYYY') expect(result).not_to include('TESTZZZ') @@ -320,7 +349,7 @@ def test_unicode write "test/b_test.rb", "sleep 1; puts 'OutputB'" write "test/c_test.rb", "sleep 1.5; puts 'OutputC'" write "test/d_test.rb", "sleep 2; puts 'OutputD'" - actual = run_tests("test", :processes => 4).scan(/Output[ABCD]/) + actual = run_tests("test", processes: 4).scan(/Output[ABCD]/) actual_sorted = [*actual[0..2].sort, actual[3]] expect(actual_sorted).to match(["OutputB", "OutputC", "OutputD", "OutputA"]) end @@ -330,11 +359,11 @@ def test_unicode write "test/long_test.rb", "puts 'this is a long test'" write "test/short_test.rb", "puts 'short test'" - group_1_result = run_tests("test", :processes => 2, :add => '--only-group 1') + group_1_result = run_tests("test", processes: 2, add: '--only-group 1') expect(group_1_result).to include("this is a long test") expect(group_1_result).not_to include("short test") - group_2_result = run_tests("test", :processes => 2, :add => '--only-group 2') + group_2_result = run_tests("test", processes: 2, add: '--only-group 2') expect(group_2_result).not_to include("this is a long test") expect(group_2_result).to include("short test") end @@ -345,7 +374,7 @@ def test_unicode it "captures seed with random failures with --verbose" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){puts "TEST1"}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){1.should == 2}}' - result = run_tests "spec --verbose", :add => "--test-options '--seed 1234'", :fail => true, :type => 'rspec' + result = run_tests "spec --verbose", add: "--test-options '--seed 1234'", fail: true, type: 'rspec' expect(result).to include("Randomized with seed 1234") expect(result).to include("bundle exec rspec spec/xxx2_spec.rb --seed 1234") end @@ -360,7 +389,7 @@ def test_unicode it "passes test options" do write "test/x1_test.rb", "require 'test/unit'; class XTest < Test::Unit::TestCase; def test_xxx; end; end" - result = run_tests("test", :add => '--test-options "-v"') + result = run_tests("test", add: '--test-options "-v"') expect(result).to include('test_xxx') # verbose output of every test end @@ -380,7 +409,7 @@ def test_unicode it "runs tests which outputs accented characters" do write "features/good1.feature", "Feature: xxx\n Scenario: xxx\n Given I print accented characters" write "features/steps/a.rb", "#encoding: utf-8\nGiven('I print accented characters'){ puts \"I tu też\" }" - result = run_tests "features", :type => "cucumber", :add => '--pattern good' + result = run_tests "features", type: "cucumber", add: '--pattern good' expect(result).to include('I tu też') end @@ -390,7 +419,7 @@ def test_unicode write "features/b.feature", "Feature: xxx\n Scenario: xxx\n Given I FAIL" write "features/steps/a.rb", "Given('I print TEST_ENV_NUMBER'){ puts \"YOUR TEST ENV IS \#{ENV['TEST_ENV_NUMBER']}!\" }" - result = run_tests "features", :type => "cucumber", :add => '--pattern good' + result = run_tests "features", type: "cucumber", add: '--pattern good' expect(result).to include('YOUR TEST ENV IS 2!') expect(result).to include('YOUR TEST ENV IS !') @@ -406,7 +435,7 @@ def test_unicode # needs sleep so that runtime loggers dont overwrite each other initially write "features/good#{i}.feature", "Feature: xxx\n Scenario: xxx\n Given I print TEST_ENV_NUMBER\n And I sleep a bit" } - run_tests "features", :type => "cucumber" + run_tests "features", type: "cucumber" expect(read(log).gsub(/\.\d+/, '').split("\n")).to match_array(["features/good0.feature:0", "features/good1.feature:0"]) end @@ -414,7 +443,7 @@ def test_unicode 2.times { |i| write "features/good#{i}.feature", "Feature: xxx\n Scenario: xxx\n Given I print TEST_ENV_NUMBER" } - result = run_tests "features", :type => "cucumber", :add => '-n 3' + result = run_tests "features", type: "cucumber", add: '-n 3' expect(result.scan(/YOUR TEST ENV IS \d?!/).sort).to eq(["YOUR TEST ENV IS !", "YOUR TEST ENV IS 2!"]) end @@ -424,13 +453,13 @@ def test_unicode write "features/pass.feature", "Feature: xxx\n Scenario: xxx\n Given I pass" write "features/fail1.feature", "Feature: xxx\n Scenario: xxx\n Given I fail" write "features/fail2.feature", "Feature: xxx\n Scenario: xxx\n Given I fail" - results = run_tests "features", :processes => 3, :type => "cucumber", :fail => true + results = run_tests "features", processes: 3, type: "cucumber", fail: true failing_scenarios = if Gem.win_platform? - ["cucumber features/fail1.feature:2 # Scenario: xxx", "cucumber features/fail2.feature:2 # Scenario: xxx"] - else - ["cucumber features/fail2.feature:2 # Scenario: xxx", "cucumber features/fail1.feature:2 # Scenario: xxx"] - end + ["cucumber features/fail1.feature:2 # Scenario: xxx", "cucumber features/fail2.feature:2 # Scenario: xxx"] + else + ["cucumber features/fail2.feature:2 # Scenario: xxx", "cucumber features/fail1.feature:2 # Scenario: xxx"] + end expect(results).to include <<-EOF.gsub(' ', '') Failing Scenarios: @@ -459,7 +488,7 @@ def test_unicode | one | | two | EOS - result = run_tests "features", :type => "cucumber", :add => "--group-by scenarios" + result = run_tests "features", type: "cucumber", add: "--group-by scenarios" expect(result).to include("2 processes for 4 scenarios") end @@ -467,14 +496,14 @@ def test_unicode write "features/good1.feature", "Feature: xxx\n Scenario: xxx\n Given I print TEST_ENV_NUMBER" write "features/good2.feature", "Feature: xxx\n Scenario: xxx\n Given I print TEST_ENV_NUMBER" - result = run_tests "features", :type => "cucumber", :add => '--group-by steps' + result = run_tests "features", type: "cucumber", add: '--group-by steps' expect(result).to include("2 processes for 2 features") end it "captures seed with random failures with --verbose" do write "features/good1.feature", "Feature: xxx\n Scenario: xxx\n Given I fail" - result = run_tests "features --verbose", :type => "cucumber", :add => '--test-options "--order random:1234"', :fail => true + result = run_tests "features --verbose", type: "cucumber", add: '--test-options "--order random:1234"', fail: true expect(result).to include("Randomized with seed 1234") expect(result).to match(%r{bundle exec cucumber "?features/good1.feature"? --order random:1234}) end @@ -497,7 +526,7 @@ class A < Spinach::FeatureSteps it "runs tests which outputs accented characters" do write "features/good1.feature", "Feature: a\n Scenario: xxx\n Given I print accented characters" write "features/steps/a.rb", "#encoding: utf-8\nclass A < Spinach::FeatureSteps\nGiven 'I print accented characters' do\n puts \"I tu też\" \n end\nend" - result = run_tests "features", :type => "spinach", :add => 'features/good1.feature' #, :add => '--pattern good' + result = run_tests "features", type: "spinach", add: 'features/good1.feature' #, :add => '--pattern good' expect(result).to include('I tu też') end @@ -506,7 +535,7 @@ class A < Spinach::FeatureSteps write "features/good2.feature", "Feature: a\n Scenario: xxx\n Given I print TEST_ENV_NUMBER" write "features/b.feature", "Feature: b\n Scenario: xxx\n Given I FAIL" #Expect this not to be run - result = run_tests "features", :type => "spinach", :add => '--pattern good' + result = run_tests "features", type: "spinach", add: '--pattern good' expect(result).to include('YOUR TEST ENV IS 2!') expect(result).to include('YOUR TEST ENV IS !') @@ -522,7 +551,7 @@ class A < Spinach::FeatureSteps # needs sleep so that runtime loggers dont overwrite each other initially write "features/good#{i}.feature", "Feature: A\n Scenario: xxx\n Given I print TEST_ENV_NUMBER\n And I sleep a bit" } - run_tests "features", :type => "spinach" + run_tests "features", type: "spinach" expect(read(log).gsub(/\.\d+/, '').split("\n")).to match_array(["features/good0.feature:0", "features/good1.feature:0"]) end @@ -530,7 +559,7 @@ class A < Spinach::FeatureSteps 2.times { |i| write "features/good#{i}.feature", "Feature: A\n Scenario: xxx\n Given I print TEST_ENV_NUMBER\n" } - result = run_tests "features", :type => "spinach", :add => '-n 3' + result = run_tests "features", type: "spinach", add: '-n 3' expect(result.scan(/YOUR TEST ENV IS \d?!/).sort).to eq(["YOUR TEST ENV IS !", "YOUR TEST ENV IS 2!"]) end From a91f7519605c4953b799185d448cc2868ad9ad8b Mon Sep 17 00:00:00 2001 From: artem Date: Tue, 7 Jul 2020 21:49:37 +0300 Subject: [PATCH 3/7] Updated test, added case check without --fail-fast --- spec/integration_spec.rb | 43 +++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 271b3176..30a2669f 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -83,14 +83,18 @@ def self.it_fails_without_any_files(type) expect(result).to include '2 processes for 2 specs, ~ 1 specs per process' end - it "fast fail in parallel" do + it "fast fail in parallel (enabled)" do + # add extra specs to verify they won't be executed + # Fail the suite at the first step, and add sleep so the tests were less flaky write 'spec/xxx1_spec.rb', 'describe("it"){it("should"){sleep 1; expect(1).to eq(2)}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST2"}}' write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST3"}}' write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST4"}}' write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST5"}}' write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST6"}}' - # set processes to false so we verify empty groups are discarded by default + # Use 2 processes so it was possible to check that all threads stop + # Use --fail-fast option for parallel tests and pass the same option to the rspec + # Use group-by found so the order of the executed specs was the same from test to test result = run_tests "spec", fail: true, type: 'rspec', @@ -98,20 +102,49 @@ def self.it_fails_without_any_files(type) add: "--group-by found --fail-fast --test-options '--fail-fast'" # test ran and gave their puts - expect(result).to include('TEST2') expect(result).to include('TEST4') + expect(result).to include('TEST5') # all results present expect(result).to include_exactly_times('1 example, 1 failure', 1) # results expect(result).to include_exactly_times('2 examples, 0 failure', 1) # results - expect(result).to include_exactly_times('3 examples, 1 failure', 1) # 1 summary + expect(result).to include_exactly_times('3 examples, 1 failure', 1) # 1 summary, verify only 3 specs were executed expect(result).to include_exactly_times(/Finished in \d+(\.\d+)? seconds/, 2) expect(result).to include_exactly_times(/Took \d+ seconds/, 1) # parallel summary - # verify empty groups are discarded. if retained then it'd say 4 processes for 2 specs + # verify that successful run would have 6 specs expect(result).to include '2 processes for 6 specs, ~ 3 specs per process' end + it "fast fail in parallel (disabled)" do + write 'spec/xxx1_spec.rb', 'describe("it"){it("should"){expect(1).to eq(2)}}' + write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){puts "TEST2"}}' + write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){puts "TEST3"}}' + write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){puts "TEST4"}}' + write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){puts "TEST5"}}' + write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){puts "TEST6"}}' + + result = run_tests "spec", + fail: true, + type: 'rspec', + processes: 2, + add: "--group-by found" + + # test ran and gave their puts + expect(result).to include('TEST2') + expect(result).to include('TEST3') + expect(result).to include('TEST4') + expect(result).to include('TEST5') + expect(result).to include('TEST6') + + # all results present + expect(result).to include_exactly_times('3 examples, 1 failure', 1) # results + expect(result).to include_exactly_times('3 examples, 0 failure', 1) # results + expect(result).to include_exactly_times('6 examples, 1 failure', 1) # 1 summary, verify all specs were executed + expect(result).to include_exactly_times(/Finished in \d+(\.\d+)? seconds/, 2) + expect(result).to include_exactly_times(/Took \d+ seconds/, 1) # parallel summary + end + it "runs tests which outputs accented characters" do write "spec/xxx_spec.rb", "#encoding: utf-8\ndescribe('it'){it('should'){puts 'Byłem tu'}}" result = run_tests "spec", type: 'rspec' From 8169807654e61c08922e4ad983f174429b1b449a Mon Sep 17 00:00:00 2001 From: artem Date: Tue, 7 Jul 2020 21:58:41 +0300 Subject: [PATCH 4/7] Updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db05cc30..7dfc3dee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ ### Added -- None +- `--fail-fast` options which stops all threads if one of them return not zero exit code. Which add possibility to stop whole suite if one test failed. Works if the option `--fail-fast` enabled for the rspec (passed to the test_options: `--test-options '--fail-fast'` or enabled at the .rspec_parallel file). ### Fixed From e81e6dcbc3b0ef4b6f5b11af0320e347e453e883 Mon Sep 17 00:00:00 2001 From: artem Date: Tue, 7 Jul 2020 22:46:55 +0300 Subject: [PATCH 5/7] Different platform separate files in to different group. So different specs will be executed and different puts will be displayed. Changed puts to be the same at all specs to assert amount of puts outputs which should be the same --- spec/integration_spec.rb | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 30a2669f..8a4c05f0 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -87,11 +87,11 @@ def self.it_fails_without_any_files(type) # add extra specs to verify they won't be executed # Fail the suite at the first step, and add sleep so the tests were less flaky write 'spec/xxx1_spec.rb', 'describe("it"){it("should"){sleep 1; expect(1).to eq(2)}}' - write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST2"}}' - write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST3"}}' - write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST4"}}' - write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST5"}}' - write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST6"}}' + write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' + write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' + write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' + write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' + write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' # Use 2 processes so it was possible to check that all threads stop # Use --fail-fast option for parallel tests and pass the same option to the rspec # Use group-by found so the order of the executed specs was the same from test to test @@ -102,8 +102,7 @@ def self.it_fails_without_any_files(type) add: "--group-by found --fail-fast --test-options '--fail-fast'" # test ran and gave their puts - expect(result).to include('TEST4') - expect(result).to include('TEST5') + expect(result).to include_exactly_times('TESTS', 2) # all results present expect(result).to include_exactly_times('1 example, 1 failure', 1) # results From e1f35ebb64de9b80b12cac3dca48b88743695cad Mon Sep 17 00:00:00 2001 From: artem Date: Tue, 14 Jul 2020 00:17:10 +0300 Subject: [PATCH 6/7] Updated indentation as was before Changed order of the command, reprint output before stopping all the processes --- lib/parallel_tests/cli.rb | 2 +- spec/integration_spec.rb | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/parallel_tests/cli.rb b/lib/parallel_tests/cli.rb index 98e8166c..4d276f71 100644 --- a/lib/parallel_tests/cli.rb +++ b/lib/parallel_tests/cli.rb @@ -44,8 +44,8 @@ def execute_in_parallel(items, num_processes, options) simulate_output_for_ci options[:serialize_stdout] do Parallel.map(items, :in_threads => num_processes) do |item| result = yield(item) - ParallelTests.stop_all_processes if result[:exit_status] != 0 && options[:fail_fast] reprint_output(result, lock.path) if options[:serialize_stdout] + ParallelTests.stop_all_processes if result[:exit_status] != 0 && options[:fail_fast] result end end diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 8a4c05f0..f43b3176 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -488,10 +488,10 @@ def test_unicode results = run_tests "features", processes: 3, type: "cucumber", fail: true failing_scenarios = if Gem.win_platform? - ["cucumber features/fail1.feature:2 # Scenario: xxx", "cucumber features/fail2.feature:2 # Scenario: xxx"] - else - ["cucumber features/fail2.feature:2 # Scenario: xxx", "cucumber features/fail1.feature:2 # Scenario: xxx"] - end + ["cucumber features/fail1.feature:2 # Scenario: xxx", "cucumber features/fail2.feature:2 # Scenario: xxx"] + else + ["cucumber features/fail2.feature:2 # Scenario: xxx", "cucumber features/fail1.feature:2 # Scenario: xxx"] + end expect(results).to include <<-EOF.gsub(' ', '') Failing Scenarios: From 2ea5bf0f3c9701fc1322c210d8b44d649a1e0726 Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Thu, 23 Jul 2020 21:20:56 -0700 Subject: [PATCH 7/7] fixups --- .github/PULL_REQUEST_TEMPLATE.md | 1 + CHANGELOG.md | 2 +- Readme.md | 11 ++-- lib/parallel_tests/cli.rb | 6 +- spec/integration_spec.rb | 103 ++++++++++++++----------------- 5 files changed, 57 insertions(+), 66 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 32fc2137..a112c405 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,3 +5,4 @@ Thank you for your contribution! - [ ] Added tests. - [ ] Added an entry to the [Changelog](../blob/master/CHANGELOG.md) if the new code introduces user-observable changes. +- [ ] Update Readme.md when cli options are changed diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dfc3dee..a2a9eb8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ ### Added -- `--fail-fast` options which stops all threads if one of them return not zero exit code. Which add possibility to stop whole suite if one test failed. Works if the option `--fail-fast` enabled for the rspec (passed to the test_options: `--test-options '--fail-fast'` or enabled at the .rspec_parallel file). +- `--fail-fast` stops all groups if one group fails. Can be used to stop all groups if one test failed by using `fail-fast` in the test-framework too (for example rspec via `--test-options '--fail-fast'` or in `.rspec_parallel`). ### Fixed diff --git a/Readme.md b/Readme.md index 182d8ab2..3463335a 100644 --- a/Readme.md +++ b/Readme.md @@ -192,7 +192,6 @@ Setup for non-rails Options are: - -n [PROCESSES] How many processes to use, default: available CPUs -p, --pattern [PATTERN] run tests matching this regex pattern --exclude-pattern [PATTERN] exclude tests matching this regex pattern @@ -222,12 +221,14 @@ Options are: --ignore-tags [PATTERN] When counting steps ignore scenarios with tags that match this pattern --nice execute test commands with low priority. --runtime-log [PATH] Location of previously recorded test runtimes - --allowed-missing Allowed percentage of missing runtimes (default = 50) + --allowed-missing [INT] Allowed percentage of missing runtimes (default = 50) --unknown-runtime [FLOAT] Use given number as unknown runtime (otherwise use average time) + --first-is-1 Use "1" as TEST_ENV_NUMBER to not reuse the default test environment + --fail-fast Stop all groups when one group fails (best used with --test-options '--fail-fast' if supported --verbose Print debug output - --verbose-process-command Print the command that will be executed by each process before it begins - --verbose-rerun-command After a process fails, print the command executed by that process - --quiet Print only test output + --verbose-process-command Displays only the command that will be executed by each process + --verbose-rerun-command When there are failures, displays the command executed by each process that failed + --quiet Print only tests output -v, --version Show Version -h, --help Show this. diff --git a/lib/parallel_tests/cli.rb b/lib/parallel_tests/cli.rb index 4d276f71..089cd838 100644 --- a/lib/parallel_tests/cli.rb +++ b/lib/parallel_tests/cli.rb @@ -42,10 +42,10 @@ def execute_in_parallel(items, num_processes, options) Tempfile.open 'parallel_tests-lock' do |lock| ParallelTests.with_pid_file do simulate_output_for_ci options[:serialize_stdout] do - Parallel.map(items, :in_threads => num_processes) do |item| + Parallel.map(items, in_threads: num_processes) do |item| result = yield(item) reprint_output(result, lock.path) if options[:serialize_stdout] - ParallelTests.stop_all_processes if result[:exit_status] != 0 && options[:fail_fast] + ParallelTests.stop_all_processes if options[:fail_fast] && result[:exit_status] != 0 result end end @@ -221,7 +221,7 @@ def parse_options!(argv) opts.on("--allowed-missing [INT]", Integer, "Allowed percentage of missing runtimes (default = 50)") { |percent| options[:allowed_missing_percent] = percent } opts.on("--unknown-runtime [FLOAT]", Float, "Use given number as unknown runtime (otherwise use average time)") { |time| options[:unknown_runtime] = time } opts.on("--first-is-1", "Use \"1\" as TEST_ENV_NUMBER to not reuse the default test environment") { options[:first_is_1] = true } - opts.on("--fail-fast", "Stop running the test suite on the first failed test") { options[:fail_fast] = true } + opts.on("--fail-fast", "Stop all groups when one group fails (best used with --test-options '--fail-fast' if supported") { options[:fail_fast] = true } opts.on("--verbose", "Print debug output") { options[:verbose] = true } opts.on("--verbose-process-command", "Displays only the command that will be executed by each process") { options[:verbose_process_command] = true } opts.on("--verbose-rerun-command", "When there are failures, displays the command executed by each process that failed") { options[:verbose_rerun_command] = true } diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index f43b3176..bfb32323 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -83,65 +83,54 @@ def self.it_fails_without_any_files(type) expect(result).to include '2 processes for 2 specs, ~ 1 specs per process' end - it "fast fail in parallel (enabled)" do - # add extra specs to verify they won't be executed - # Fail the suite at the first step, and add sleep so the tests were less flaky - write 'spec/xxx1_spec.rb', 'describe("it"){it("should"){sleep 1; expect(1).to eq(2)}}' - write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' - write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' - write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' - write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' - write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TESTS"}}' - # Use 2 processes so it was possible to check that all threads stop - # Use --fail-fast option for parallel tests and pass the same option to the rspec - # Use group-by found so the order of the executed specs was the same from test to test - result = run_tests "spec", - fail: true, - type: 'rspec', - processes: 2, - add: "--group-by found --fail-fast --test-options '--fail-fast'" + describe "--fail-fast" do + def run_tests(test_option: nil) + super( + "spec", + fail: true, + type: 'rspec', + processes: 2, + # group-by + order for stable execution ... doc and verbose to ease debugging + add: "--group-by found --verbose --fail-fast --test-options '--format doc --order defined #{test_option}'" + ) + end - # test ran and gave their puts - expect(result).to include_exactly_times('TESTS', 2) + before do + write 'spec/xxx1_spec.rb', 'describe("T1"){it("E1"){puts "YE" + "S"; sleep 0.5; expect(1).to eq(2)}}' # group 1 executed + write 'spec/xxx2_spec.rb', 'describe("T2"){it("E2"){sleep 1; puts "OK"}}' # group 2 executed + write 'spec/xxx3_spec.rb', 'describe("T3"){it("E3"){puts "NO3"}}' # group 1 skipped + write 'spec/xxx4_spec.rb', 'describe("T4"){it("E4"){puts "NO4"}}' # group 2 skipped + write 'spec/xxx5_spec.rb', 'describe("T5"){it("E5"){puts "NO5"}}' # group 1 skipped + write 'spec/xxx6_spec.rb', 'describe("T6"){it("E6"){puts "NO6"}}' # group 2 skipped + end - # all results present - expect(result).to include_exactly_times('1 example, 1 failure', 1) # results - expect(result).to include_exactly_times('2 examples, 0 failure', 1) # results - expect(result).to include_exactly_times('3 examples, 1 failure', 1) # 1 summary, verify only 3 specs were executed - expect(result).to include_exactly_times(/Finished in \d+(\.\d+)? seconds/, 2) - expect(result).to include_exactly_times(/Took \d+ seconds/, 1) # parallel summary + it "can fail fast on a single test" do + result = run_tests(test_option: "--fail-fast") - # verify that successful run would have 6 specs - expect(result).to include '2 processes for 6 specs, ~ 3 specs per process' - end + expect(result).to include_exactly_times("YES", 1) + expect(result).to include_exactly_times("OK", 1) # is allowed to finish but no new test is started after + expect(result).to_not include("NO") - it "fast fail in parallel (disabled)" do - write 'spec/xxx1_spec.rb', 'describe("it"){it("should"){expect(1).to eq(2)}}' - write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){puts "TEST2"}}' - write 'spec/xxx3_spec.rb', 'describe("it"){it("should"){puts "TEST3"}}' - write 'spec/xxx4_spec.rb', 'describe("it"){it("should"){puts "TEST4"}}' - write 'spec/xxx5_spec.rb', 'describe("it"){it("should"){puts "TEST5"}}' - write 'spec/xxx6_spec.rb', 'describe("it"){it("should"){puts "TEST6"}}' + expect(result).to include_exactly_times('1 example, 1 failure', 1) # rspec group 1 + expect(result).to include_exactly_times('1 example, 0 failure', 1) # rspec group 2 + expect(result).to include_exactly_times('2 examples, 1 failure', 1) # parallel_rspec summary + + expect(result).to include '2 processes for 6 specs, ~ 3 specs per process' + end - result = run_tests "spec", - fail: true, - type: 'rspec', - processes: 2, - add: "--group-by found" + it "can fail fast on a single group" do + result = run_tests - # test ran and gave their puts - expect(result).to include('TEST2') - expect(result).to include('TEST3') - expect(result).to include('TEST4') - expect(result).to include('TEST5') - expect(result).to include('TEST6') + expect(result).to include_exactly_times("YES", 1) + expect(result).to include_exactly_times("OK", 1) # is allowed to finish but no new test is started after + expect(result).to include_exactly_times("NO", 2) - # all results present - expect(result).to include_exactly_times('3 examples, 1 failure', 1) # results - expect(result).to include_exactly_times('3 examples, 0 failure', 1) # results - expect(result).to include_exactly_times('6 examples, 1 failure', 1) # 1 summary, verify all specs were executed - expect(result).to include_exactly_times(/Finished in \d+(\.\d+)? seconds/, 2) - expect(result).to include_exactly_times(/Took \d+ seconds/, 1) # parallel summary + expect(result).to include_exactly_times('3 examples, 1 failure', 1) # rspec group 1 + expect(result).to include_exactly_times('1 example, 0 failure', 1) # rspec group 2 + expect(result).to include_exactly_times('4 examples, 1 failure', 1) # parallel_rspec summary + + expect(result).to include '2 processes for 6 specs, ~ 3 specs per process' + end end it "runs tests which outputs accented characters" do @@ -160,11 +149,11 @@ def test_unicode end end EOF + # Need to tell Ruby to default to utf-8 to simulate environments where # this is set. (Otherwise, it defaults to nil and the undefined conversion # issue doesn't come up.) - result = run_tests('test', fail: true, - export: {'RUBYOPT' => 'Eutf-8:utf-8'}) + result = run_tests('test', fail: true, export: {'RUBYOPT' => 'Eutf-8:utf-8'}) expect(result).to include('¯\_(ツ)_/¯') end @@ -346,11 +335,11 @@ def test_unicode it "runs with PARALLEL_TEST_PROCESSORS processes" do skip if RUBY_PLATFORM == "java" # execution expired issue on JRuby processes = 5 - processes.times { |i| + processes.times do |i| write "spec/x#{i}_spec.rb", "puts %{ENV-\#{ENV['TEST_ENV_NUMBER']}-}" - } + end result = run_tests( - "spec", export: {"PARALLEL_TEST_PROCESSORS" => processes.to_s}, processes: processes, type: 'rspec' + "spec", export: {"PARALLEL_TEST_PROCESSORS" => processes.to_s}, processes: processes, type: 'rspec' ) expect(result.scan(/ENV-.?-/)).to match_array(["ENV--", "ENV-2-", "ENV-3-", "ENV-4-", "ENV-5-"]) end