From 0739982f9da501e7a71d84e116e7aa21ee560610 Mon Sep 17 00:00:00 2001 From: Koichi ITO Date: Fri, 23 Jul 2021 22:32:25 +0900 Subject: [PATCH] Parallel static analysis by default Modern PCs have multi-core. RuboCop has `--parallel` option, but it is not used by default. This means that there is extra core resources for static analysis because multi-core is not being used effectively. This PR will be parallel processing by default, so RuboCop will run faster in proportion to number of cores. Options that can be parallel processing, such as `--only`, `--except`, and others have the same default behavior. However, there are options that cannot be parallel processing, such as auto-correction and others, so they are serial processing as before. `--parallel` can be a hidden option for some users. This change is expected to reduce user's wait time for running RuboCop. Below is an example on my development PC. Before: ```console % cd path/to/rubocop ./exe/rubocop --display-time (snip) 1293 files inspected, no offenses detected Finished in 61.09493000002112 seconds ``` After: ```console % cd path/to/rubocop ./exe/rubocop --display-time (snip) 1293 files inspected, no offenses detected Finished in 10.13813699997263 seconds ``` If user's PC has a lot of cores, memory and RuboCop targets a lot of files to inspect, it will be effective. One concern is that parallel processing uses more memory. For example, Ruby's MJIT is off by default as an example of concerning about memory consumption in a low memory production runtime (e.g. Heroku's minimum Dyno). However, RuboCop is a development tool, I assume that it will be fine in most cases. If user want to use it in a low memory production runtime, user can specify `--no-parallel`. Note: JRuby and Windows are not supported based on #4537. --- Rakefile | 7 +-- ...new_parallel_static_analysis_by_default.md | 1 + lib/rubocop/cli.rb | 18 ++++++ lib/rubocop/options.rb | 2 +- spec/rubocop/cli_spec.rb | 63 +++++++++++++++++++ spec/rubocop/options_spec.rb | 2 +- 6 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 changelog/new_parallel_static_analysis_by_default.md diff --git a/Rakefile b/Rakefile index 88a41dc9ccc..60d03d7719e 100644 --- a/Rakefile +++ b/Rakefile @@ -22,12 +22,7 @@ require 'rubocop/rake_task' Dir['tasks/**/*.rake'].each { |t| load t } desc 'Run RuboCop over itself' -RuboCop::RakeTask.new(:internal_investigation).tap do |task| - if RUBY_ENGINE == 'ruby' && - !/mswin|msys|mingw|cygwin|bccwin|wince|emc/.match?(RbConfig::CONFIG['host_os']) - task.options = %w[--parallel] - end -end +RuboCop::RakeTask.new(:internal_investigation) task default: %i[documentation_syntax_check spec ascii_spec internal_investigation] diff --git a/changelog/new_parallel_static_analysis_by_default.md b/changelog/new_parallel_static_analysis_by_default.md new file mode 100644 index 00000000000..974762154ef --- /dev/null +++ b/changelog/new_parallel_static_analysis_by_default.md @@ -0,0 +1 @@ +* [#10000](https://github.com/rubocop/rubocop/pull/10000): Parallel static analysis by default. ([@koic][]) diff --git a/lib/rubocop/cli.rb b/lib/rubocop/cli.rb index 7553f203fec..3666a09c997 100644 --- a/lib/rubocop/cli.rb +++ b/lib/rubocop/cli.rb @@ -8,6 +8,11 @@ class CLI STATUS_OFFENSES = 1 STATUS_ERROR = 2 STATUS_INTERRUPTED = 128 + Signal.list['INT'] + DEFAULT_PARALLEL_OPTIONS = %i[ + color debug display_style_guide display_time display_only_fail_level_offenses + display_only_failed except extra_details fail_level fix_layout format + ignore_disable_comments lint only only_guide_cops require safe + ].freeze class Finished < RuntimeError; end @@ -37,6 +42,7 @@ def run(args = ARGV) else act_on_options validate_options_vs_config + parallel_by_default! apply_default_formatter execute_runners end @@ -84,6 +90,18 @@ def validate_options_vs_config 'with AllCops: UseCache: false is not allowed.' end + def parallel_by_default! + # See https://github.com/rubocop/rubocop/pull/4537 for JRuby and Windows constraints. + return if RUBY_ENGINE != 'ruby' || RuboCop::Platform.windows? + + if (@options.keys - DEFAULT_PARALLEL_OPTIONS).empty? && + @config_store.for_pwd.for_all_cops['UseCache'] != false + puts 'Use parallel by default.' if @options[:debug] + + @options[:parallel] = true + end + end + def act_on_options set_options_to_config_loader diff --git a/lib/rubocop/options.rb b/lib/rubocop/options.rb index 6ad40864989..6a598cf8414 100644 --- a/lib/rubocop/options.rb +++ b/lib/rubocop/options.rb @@ -492,7 +492,7 @@ module OptionsHelp version: 'Display version.', verbose_version: 'Display verbose version.', parallel: ['Use available CPUs to execute inspection in', - 'parallel. Default is false.'], + 'parallel. Default is true.'], stdin: ['Pipe source from STDIN, using FILE in offense', 'reports. This is useful for editor integration.'], init: 'Generate a .rubocop.yml file in the current directory.' diff --git a/spec/rubocop/cli_spec.rb b/spec/rubocop/cli_spec.rb index 75a4e43bfd9..34fc49d07c7 100644 --- a/spec/rubocop/cli_spec.rb +++ b/spec/rubocop/cli_spec.rb @@ -197,6 +197,69 @@ def and_with_args end end + if RUBY_ENGINE == 'ruby' && !RuboCop::Platform.windows? + # NOTE: It has been tested for parallelism with `--debug` option. + # In other words, even if no option is specified, it will be parallelized by default. + describe 'when parallel static by default' do + context 'when specifying `--debug` option only`' do + it 'fails with an error message' do + create_file('example1.rb', <<~RUBY) + # frozen_string_literal: true + + puts 'hello' + RUBY + expect(cli.run(['--debug'])).to eq(0) + expect($stdout.string).to include('Use parallel by default.') + end + end + + # NOTE: Cannot be auto-corrected with `parallel`. + context 'when specifying `--debug` and `-a` options`' do + it 'fails with an error message' do + create_file('example1.rb', <<~RUBY) + # frozen_string_literal: true + + puts 'hello' + RUBY + expect(cli.run(['--debug', '-a'])).to eq(0) + expect($stdout.string).not_to include('Use parallel by default.') + end + end + + context 'when setting `UseCache: true`' do + it 'fails with an error message' do + create_file('example.rb', <<~RUBY) + # frozen_string_literal: true + + puts 'hello' + RUBY + create_file('.rubocop.yml', <<~YAML) + AllCops: + UseCache: true + YAML + expect(cli.run(['--debug'])).to eq(0) + expect($stdout.string).to include('Use parallel by default.') + end + end + + context 'when setting `UseCache: false`' do + it 'fails with an error message' do + create_file('example.rb', <<~RUBY) + # frozen_string_literal: true + + puts 'hello' + RUBY + create_file('.rubocop.yml', <<~YAML) + AllCops: + UseCache: false + YAML + expect(cli.run(['--debug'])).to eq(0) + expect($stdout.string).not_to include('Use parallel by default.') + end + end + end + end + describe 'rubocop:disable comment' do it 'can disable all cops in a code section' do src = ['# rubocop:disable all', diff --git a/spec/rubocop/options_spec.rb b/spec/rubocop/options_spec.rb index 4370660837b..4221a6e00c1 100644 --- a/spec/rubocop/options_spec.rb +++ b/spec/rubocop/options_spec.rb @@ -138,7 +138,7 @@ def abs(path) -v, --version Display version. -V, --verbose-version Display verbose version. -P, --[no-]parallel Use available CPUs to execute inspection in - parallel. Default is false. + parallel. Default is true. -l, --lint Run only lint cops. -x, --fix-layout Run only layout cops, with auto-correct on. -s, --stdin FILE Pipe source from STDIN, using FILE in offense