forked from grosser/parallel_tests
/
cli.rb
367 lines (307 loc) · 14.4 KB
/
cli.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
require 'optparse'
require 'tempfile'
require 'parallel_tests'
require 'shellwords'
require 'pathname'
module ParallelTests
class CLI
def run(argv)
Signal.trap("INT") { handle_interrupt }
options = parse_options!(argv)
ENV['DISABLE_SPRING'] ||= '1'
num_processes = ParallelTests.determine_number_of_processes(options[:count])
num_processes = num_processes * (options[:multiply] || 1)
options[:first_is_1] ||= first_is_1?
if options[:execute]
execute_shell_command_in_parallel(options[:execute], num_processes, options)
else
run_tests_in_parallel(num_processes, options)
end
end
private
def handle_interrupt
@graceful_shutdown_attempted ||= false
Kernel.exit if @graceful_shutdown_attempted
# The Pid class's synchronize method can't be called directly from a trap
# Using Thread workaround https://github.com/ddollar/foreman/issues/332
Thread.new { ParallelTests.stop_all_processes }
@graceful_shutdown_attempted = true
end
def execute_in_parallel(items, num_processes, options)
Tempfile.open 'parallel_tests-lock' do |lock|
ParallelTests.with_pid_file do
simulate_output_for_ci options[:serialize_stdout] do
Parallel.map(items, in_threads: num_processes) do |item|
result = yield(item)
reprint_output(result, lock.path) if options[:serialize_stdout]
ParallelTests.stop_all_processes if options[:fail_fast] && result[:exit_status] != 0
result
end
end
end
end
end
def run_tests_in_parallel(num_processes, options)
test_results = nil
run_tests_proc = -> {
groups = @runner.tests_in_groups(options[:files], num_processes, options)
groups.reject! &:empty?
test_results = if options[:only_group]
groups_to_run = options[:only_group].collect{|i| groups[i - 1]}.compact
report_number_of_tests(groups_to_run) unless options[:quiet]
execute_in_parallel(groups_to_run, groups_to_run.size, options) do |group|
run_tests(group, groups_to_run.index(group), 1, options)
end
else
report_number_of_tests(groups) unless options[:quiet]
execute_in_parallel(groups, groups.size, options) do |group|
run_tests(group, groups.index(group), num_processes, options)
end
end
report_results(test_results, options) unless options[:quiet]
}
if options[:quiet]
run_tests_proc.call
else
report_time_taken(&run_tests_proc)
end
abort final_fail_message if any_test_failed?(test_results)
end
def run_tests(group, process_number, num_processes, options)
if group.empty?
{:stdout => '', :exit_status => 0, :command => '', :seed => nil}
else
@runner.run_tests(group, process_number, num_processes, options)
end
end
def reprint_output(result, lockfile)
lock(lockfile) do
$stdout.puts
$stdout.puts result[:stdout]
$stdout.flush
end
end
def lock(lockfile)
File.open(lockfile) do |lock|
begin
lock.flock File::LOCK_EX
yield
ensure
# This shouldn't be necessary, but appears to be
lock.flock File::LOCK_UN
end
end
end
def report_results(test_results, options)
results = @runner.find_results(test_results.map { |result| result[:stdout] }*"")
puts ""
puts @runner.summarize_results(results)
report_failure_rerun_commmand(test_results, options)
end
def report_failure_rerun_commmand(test_results, options)
failing_sets = test_results.reject { |r| r[:exit_status] == 0 }
return if failing_sets.none?
if options[:verbose] || options[:verbose_rerun_command]
puts "\n\nTests have failed for a parallel_test group. Use the following command to run the group again:\n\n"
failing_sets.each do |failing_set|
command = failing_set[:command]
command = command.gsub(/;export [A-Z_]+;/, ' ') # remove ugly export statements
command = @runner.command_with_seed(command, failing_set[:seed]) if failing_set[:seed]
puts command
end
end
end
def report_number_of_tests(groups)
name = @runner.test_file_name
num_processes = groups.size
num_tests = groups.map(&:size).inject(0, :+)
tests_per_process = (num_processes == 0 ? 0 : num_tests / num_processes)
puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process"
end
#exit with correct status code so rake parallel:test && echo 123 works
def any_test_failed?(test_results)
test_results.any? { |result| result[:exit_status] != 0 }
end
def parse_options!(argv)
options = {}
OptionParser.new do |opts|
opts.banner = <<-BANNER.gsub(/^ /, '')
Run all tests in parallel, giving each process ENV['TEST_ENV_NUMBER'] ('', '2', '3', ...)
[optional] Only selected files & folders:
parallel_test test/bar test/baz/xxx_text.rb
[optional] Pass test-options and files via `--`:
parallel_test -- -t acceptance -f progress -- spec/foo_spec.rb spec/acceptance
Options are:
BANNER
opts.on("-n [PROCESSES]", Integer, "How many processes to use, default: available CPUs") { |n| options[:count] = n }
opts.on("-p", "--pattern [PATTERN]", "run tests matching this regex pattern") { |pattern| options[:pattern] = /#{pattern}/ }
opts.on("--exclude-pattern", "--exclude-pattern [PATTERN]", "exclude tests matching this regex pattern") { |pattern| options[:exclude_pattern] = /#{pattern}/ }
opts.on("--group-by [TYPE]", <<-TEXT.gsub(/^ /, '')
group tests by:
found - order of finding files
steps - number of cucumber/spinach steps
scenarios - individual cucumber scenarios
filesize - by size of the file
runtime - info from runtime log
default - runtime when runtime log is filled otherwise filesize
TEXT
) { |type| options[:group_by] = type.to_sym }
opts.on("-m [FLOAT]", "--multiply-processes [FLOAT]", Float, "use given number as a multiplier of processes to run") { |multiply| options[:multiply] = multiply }
opts.on("-s [PATTERN]", "--single [PATTERN]",
"Run all matching files in the same process") do |pattern|
options[:single_process] ||= []
options[:single_process] << /#{pattern}/
end
opts.on("-i", "--isolate",
"Do not run any other tests in the group used by --single(-s)") do |pattern|
options[:isolate] = true
options[:isolate_count] = 1
end
opts.on("--isolate-n [PROCESSES]",
Integer,
"Number of processes to use for isolated singles, default: 1. Do not set this without setting --isolate.") do |n|
# isolate_count is dependent on isolate being set.
abort "Don't use isolate-n without isolate" unless options[:isolate]
if n >= ParallelTests.determine_number_of_processes(options[:count])
abort 'Number of isolated processes must be less than total the number of processes'
end
options[:isolate_count] = n
end
opts.on("--only-group INT[, INT]", Array) { |groups| options[:only_group] = groups.map(&:to_i) }
opts.on("-e", "--exec [COMMAND]", "execute this code parallel and with ENV['TEST_ENV_NUMBER']") { |path| options[:execute] = path }
opts.on("-o", "--test-options '[OPTIONS]'", "execute test commands with those options") { |arg| options[:test_options] = arg.lstrip }
opts.on("-t", "--type [TYPE]", "test(default) / rspec / cucumber / spinach") do |type|
begin
@runner = load_runner(type)
rescue NameError, LoadError => e
puts "Runner for `#{type}` type has not been found! (#{e})"
abort
end
end
opts.on("--suffix [PATTERN]", <<-TEXT.gsub(/^ /, '')
override built in test file pattern (should match suffix):
'_spec\.rb$' - matches rspec files
'_(test|spec).rb$' - matches test or spec files
TEXT
) { |pattern| options[:suffix] = /#{pattern}/ }
opts.on("--serialize-stdout", "Serialize stdout output, nothing will be written until everything is done") { options[:serialize_stdout] = true }
opts.on("--prefix-output-with-test-env-number", "Prefixes test env number to the output when not using --serialize-stdout") { options[:prefix_output_with_test_env_number] = true }
opts.on("--combine-stderr", "Combine stderr into stdout, useful in conjunction with --serialize-stdout") { options[:combine_stderr] = true }
opts.on("--non-parallel", "execute same commands but do not in parallel, needs --exec") { options[:non_parallel] = true }
opts.on("--no-symlinks", "Do not traverse symbolic links to find test files") { options[:symlinks] = false }
opts.on('--ignore-tags [PATTERN]', 'When counting steps ignore scenarios with tags that match this pattern') { |arg| options[:ignore_tag_pattern] = arg }
opts.on("--nice", "execute test commands with low priority.") { options[:nice] = true }
opts.on("--runtime-log [PATH]", "Location of previously recorded test runtimes") { |path| options[:runtime_log] = path }
opts.on("--allowed-missing [INT]", Integer, "Allowed percentage of missing runtimes (default = 50)") { |percent| options[:allowed_missing_percent] = percent }
opts.on("--unknown-runtime [FLOAT]", Float, "Use given number as unknown runtime (otherwise use average time)") { |time| options[:unknown_runtime] = time }
opts.on("--first-is-1", "Use \"1\" as TEST_ENV_NUMBER to not reuse the default test environment") { options[:first_is_1] = true }
opts.on("--fail-fast", "Stop all groups when one group fails (best used with --test-options '--fail-fast' if supported") { options[:fail_fast] = true }
opts.on("--verbose", "Print debug output") { options[:verbose] = true }
opts.on("--verbose-process-command", "Displays only the command that will be executed by each process") { options[:verbose_process_command] = true }
opts.on("--verbose-rerun-command", "When there are failures, displays the command executed by each process that failed") { options[:verbose_rerun_command] = true }
opts.on("--quiet", "Print only tests output") { options[:quiet] = true }
opts.on("-v", "--version", "Show Version") { puts ParallelTests::VERSION; exit }
opts.on("-h", "--help", "Show this.") { puts opts; exit }
end.parse!(argv)
if options[:verbose] && options[:quiet]
raise "Both options are mutually exclusive: verbose & quiet"
end
if options[:count] == 0
options.delete(:count)
options[:non_parallel] = true
end
files, remaining = extract_file_paths(argv)
unless options[:execute]
abort "Pass files or folders to run" unless files.any?
options[:files] = files.map { |file_path| Pathname.new(file_path).cleanpath.to_s }
end
append_test_options(options, remaining)
options[:group_by] ||= :filesize if options[:only_group]
raise "--group-by found and --single-process are not supported" if options[:group_by] == :found and options[:single_process]
allowed = [:filesize, :runtime, :found]
if !allowed.include?(options[:group_by]) && options[:only_group]
raise "--group-by #{allowed.join(" or ")} is required for --only-group"
end
options
end
def extract_file_paths(argv)
dash_index = argv.rindex("--")
file_args_at = (dash_index || -1) + 1
[argv[file_args_at..-1], argv[0...(dash_index || 0)]]
end
def extract_test_options(argv)
dash_index = argv.index("--") || -1
argv[dash_index+1..-1]
end
def append_test_options(options, argv)
new_opts = extract_test_options(argv)
return if new_opts.empty?
prev_and_new = [options[:test_options], new_opts.shelljoin]
options[:test_options] = prev_and_new.compact.join(' ')
end
def load_runner(type)
require "parallel_tests/#{type}/runner"
runner_classname = type.split("_").map(&:capitalize).join.sub("Rspec", "RSpec")
klass_name = "ParallelTests::#{runner_classname}::Runner"
klass_name.split('::').inject(Object) { |x, y| x.const_get(y) }
end
def execute_shell_command_in_parallel(command, num_processes, options)
runs = if options[:only_group]
options[:only_group].map{|g| g - 1}
else
(0...num_processes).to_a
end
results = if options[:non_parallel]
ParallelTests.with_pid_file do
runs.map do |i|
ParallelTests::Test::Runner.execute_command(command, i, num_processes, options)
end
end
else
execute_in_parallel(runs, runs.size, options) do |i|
ParallelTests::Test::Runner.execute_command(command, i, num_processes, options)
end
end.flatten
abort if results.any? { |r| r[:exit_status] != 0 }
end
def report_time_taken
seconds = ParallelTests.delta { yield }.to_i
puts "\nTook #{seconds} seconds#{detailed_duration(seconds)}"
end
def detailed_duration(seconds)
parts = [ seconds / 3600, seconds % 3600 / 60, seconds % 60 ].drop_while(&:zero?)
return if parts.size < 2
parts = parts.map { |i| "%02d" % i }.join(':').sub(/^0/, '')
" (#{parts})"
end
def final_fail_message
fail_message = "Tests Failed"
fail_message = "\e[31m#{fail_message}\e[0m" if use_colors?
fail_message
end
def use_colors?
$stdout.tty?
end
def first_is_1?
val = ENV["PARALLEL_TEST_FIRST_IS_1"]
['1', 'true'].include?(val)
end
# CI systems often fail when there is no output for a long time, so simulate some output
def simulate_output_for_ci(simulate)
if simulate
progress_indicator = Thread.new do
interval = Float(ENV.fetch('PARALLEL_TEST_HEARTBEAT_INTERVAL', 60))
loop do
sleep interval
print '.'
end
end
test_results = yield
progress_indicator.exit
test_results
else
yield
end
end
end
end