From 2d41658b470af26b250024ffb10e541984b95e82 Mon Sep 17 00:00:00 2001 From: Nathan Broadbent Date: Sun, 8 Dec 2019 03:12:24 +0700 Subject: [PATCH 1/4] Fix flaky test by removing the .ordered expectation, since the tests are run in parallel threads and there are no guarantees about the execution order --- spec/parallel_tests/cli_spec.rb | 4 ++-- spec/rails_spec.rb | 2 -- spec/spec_helper.rb | 6 ------ 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/spec/parallel_tests/cli_spec.rb b/spec/parallel_tests/cli_spec.rb index 1a373aa0..1fa9e242 100644 --- a/spec/parallel_tests/cli_spec.rb +++ b/spec/parallel_tests/cli_spec.rb @@ -315,8 +315,8 @@ def self.it_prints_nothing_about_rerun_commands(options) it "run twice with multiple groups" do skip "fails on jruby" if RUBY_PLATFORM == "java" options = common_options.merge(count: 3, only_group: [2,3]) - expect(subject).to receive(:run_tests).once.ordered.with(['ccc', 'ddd'], 0, 1, options).and_return(results) - expect(subject).to receive(:run_tests).once.ordered.with(['eee', 'fff'], 1, 1, options).and_return(results) + expect(subject).to receive(:run_tests).once.with(['ccc', 'ddd'], 0, 1, options).and_return(results) + expect(subject).to receive(:run_tests).once.with(['eee', 'fff'], 1, 1, options).and_return(results) subject.run(['test', '-n', '3', '--only-group', '2,3', '-t', 'my_test_runner']) end end diff --git a/spec/rails_spec.rb b/spec/rails_spec.rb index 78a7b53b..c9391b50 100644 --- a/spec/rails_spec.rb +++ b/spec/rails_spec.rb @@ -1,8 +1,6 @@ require 'spec_helper' describe 'rails' do - let(:test_timeout) { 420 } # this can take very long on fresh bundle ... - def sh(command, options={}) result = '' IO.popen(options.fetch(:environment, {}), command, err: [:child, :out]) do |io| diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 99426980..ef40f50b 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -188,12 +188,6 @@ def setup_runtime_log config.raise_errors_for_deprecations! - # sometimes stuff hangs -> do not hang everything - config.include(Module.new {def test_timeout;30;end }) - config.around do |example| - Timeout.timeout(test_timeout, &example) - end - config.after do ENV.delete "PARALLEL_TEST_GROUPS" ENV.delete "PARALLEL_TEST_PROCESSORS" From 550eafbe2736dcdaba6bb7bea9ddcf5c14cd3914 Mon Sep 17 00:00:00 2001 From: Nathan Broadbent Date: Sun, 8 Dec 2019 03:18:50 +0700 Subject: [PATCH 2/4] Cache gems in spec/fixtures/{rails51,rails52}/vendor/bundle to speed up CI builds --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0fe929ff..be05baa7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,11 @@ rvm: branches: only: [master] sudo: false -cache: bundler +cache: + bundler: true + directories: + - spec/fixtures/rails51/vendor/bundle + - spec/fixtures/rails52/vendor/bundle matrix: allow_failures: - rvm: ruby-head From a793d8fa8594b51ce6e603d97e6336911deca538 Mon Sep 17 00:00:00 2001 From: Nathan Broadbent Date: Sun, 8 Dec 2019 16:56:49 +0700 Subject: [PATCH 3/4] Fix another flaky test by adding a "TEST" prefix to more of the output expectations. (Failing build: https://travis-ci.org/grosser/parallel_tests/jobs/622216736?utm_medium=notification&utm_source=github_status) --- spec/integration_spec.rb | 46 ++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 3fc3aecd..45cdc63f 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -229,7 +229,7 @@ def test_unicode it "runs with --group-by found" do # it only tests that it does not blow up, as it did before fixing... - write "spec/x1_spec.rb", "puts '111'" + write "spec/x1_spec.rb", "puts 'TEST111'" run_tests "spec", :type => 'rspec', :add => '--group-by found' end @@ -254,27 +254,27 @@ def test_unicode end it "runs with files that have spaces" do - write "test/xxx _test.rb", 'puts "YES"' + write "test/xxx _test.rb", 'puts "TEST_SUCCESS"' result = run_tests("test", processes: 2, type: 'test') - expect(result).to include "YES" + expect(result).to include "TEST_SUCCESS" end it "uses relative paths for easy copying" do - write "test/xxx_test.rb", 'puts "YES"' + write "test/xxx_test.rb", 'puts "Test output: YES"' result = run_tests("test", processes: 2, type: 'test', add: '--verbose') - expect(result).to include "YES" + expect(result).to include "Test output: YES" expect(result).to include "[test/xxx_test.rb]" expect(result).not_to include Dir.pwd end it "can run with given files" do - write "spec/x1_spec.rb", "puts '111'" - write "spec/x2_spec.rb", "puts '222'" - write "spec/x3_spec.rb", "puts '333'" + write "spec/x1_spec.rb", "puts 'TEST111'" + write "spec/x2_spec.rb", "puts 'TEST222'" + write "spec/x3_spec.rb", "puts 'TEST333'" result = run_tests "spec/x1_spec.rb spec/x3_spec.rb", :type => 'rspec' - expect(result).to include('111') - expect(result).to include('333') - expect(result).not_to include('222') + expect(result).to include('TEST111') + expect(result).to include('TEST333') + expect(result).not_to include('TEST222') end it "can run with test-options" do @@ -297,23 +297,23 @@ def test_unicode end it "filters test by given pattern and relative paths" do - write "spec/x_spec.rb", "puts 'XXX'" - write "spec/y_spec.rb", "puts 'YYY'" - write "spec/z_spec.rb", "puts 'ZZZ'" + write "spec/x_spec.rb", "puts 'TESTXXX'" + write "spec/y_spec.rb", "puts 'TESTYYY'" + write "spec/z_spec.rb", "puts 'TESTZZZ'" result = run_tests "spec", :add => "-p '^spec/(x|z)'", :type => "rspec" - expect(result).to include('XXX') - expect(result).not_to include('YYY') - expect(result).to include('ZZZ') + expect(result).to include('TESTXXX') + expect(result).not_to include('TESTYYY') + expect(result).to include('TESTZZZ') end it "excludes test by given pattern and relative paths" do - write "spec/x_spec.rb", "puts 'XXX'" - write "spec/acceptance/y_spec.rb", "puts 'YYY'" - write "spec/integration/z_spec.rb", "puts 'ZZZ'" + write "spec/x_spec.rb", "puts 'TESTXXX'" + write "spec/acceptance/y_spec.rb", "puts 'TESTYYY'" + write "spec/integration/z_spec.rb", "puts 'TESTZZZ'" result = run_tests "spec", :add => "--exclude-pattern 'spec/(integration|acceptance)'", :type => "rspec" - expect(result).to include('XXX') - expect(result).not_to include('YYY') - expect(result).not_to include('ZZZ') + expect(result).to include('TESTXXX') + expect(result).not_to include('TESTYYY') + expect(result).not_to include('TESTZZZ') end it "can wait_for_other_processes_to_finish" do From e48578eb0075a02f384a47637fc48bdd35cdce37 Mon Sep 17 00:00:00 2001 From: Nathan Broadbent Date: Sun, 8 Dec 2019 18:29:03 +0700 Subject: [PATCH 4/4] Decrease PARALLEL_TEST_HEARTBEAT_INTERVAL to fix occasional test failure (example: https://ci.appveyor.com/project/grosser/parallel-tests/builds/29391856/job/8j2dqxd7duf10gra) --- spec/integration_spec.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 45cdc63f..71404cda 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -160,15 +160,13 @@ def test_unicode it "can show simulated output when serializing stdout" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){sleep 0.5; puts "TEST1"}}' write 'spec/xxx2_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST2"}}' - result = run_tests "spec", :type => 'rspec', :add => "--serialize-stdout", export: {'PARALLEL_TEST_HEARTBEAT_INTERVAL' => '0.1'} - + result = run_tests "spec", :type => 'rspec', :add => "--serialize-stdout", export: {'PARALLEL_TEST_HEARTBEAT_INTERVAL' => '0.01'} expect(result).to match(/\.{4}.*TEST1.*\.{4}.*TEST2/m) end it "can show simulated output preceded by command when serializing stdout with verbose option" do write 'spec/xxx_spec.rb', 'describe("it"){it("should"){sleep 1; puts "TEST1"}}' - result = run_tests "spec --verbose", :type => 'rspec', :add => "--serialize-stdout", export: {'PARALLEL_TEST_HEARTBEAT_INTERVAL' => '0.2'} - + result = run_tests "spec --verbose", :type => 'rspec', :add => "--serialize-stdout", export: {'PARALLEL_TEST_HEARTBEAT_INTERVAL' => '0.02'} expect(result).to match(/\.{5}.*\nbundle exec rspec spec\/xxx_spec\.rb\nTEST1/m) end