More fine-grained control (using [conf]) for build parallelization (#…

…8665)

* first shot

* format the strings

* package_id change due to msbuild property name change

* do not fallback to cpu_count

* remove parallel from constructor

* remove change not needed

* add unittesting

* check once for parallel

* Update conans/test/unittests/tools/microsoft/test_msbuild.py

Co-authored-by: James <james@conan.io>

Co-authored-by: James <james@conan.io>

conan/tools/cmake/cmake.py

@@ -3,13 +3,15 @@
 
 from conan.tools.cmake.base import CMakeToolchainBase
 from conan.tools.cmake.utils import get_generator, is_multi_configuration
-from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg
+from conan.tools.gnu.make import make_jobs_cmd_line_arg
+from conan.tools.meson.meson import ninja_jobs_cmd_line_arg
+from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg, \
+    msbuild_max_cpu_count_cmd_line_arg
 from conans.client import tools
 from conans.client.build import join_arguments
 from conans.client.tools.files import chdir
 from conans.client.tools.oss import cpu_count, args_to_string
 from conans.errors import ConanException
-from conans.model.version import Version
 from conans.util.conan_v2_mode import conan_v2_error
 from conans.util.files import mkdir
 
@@ -23,15 +25,28 @@ def _validate_recipe(conanfile):
 
 def _cmake_cmd_line_args(conanfile, generator, parallel):
     args = []
-    compiler_version = conanfile.settings.get_safe("compiler.version")
-    if generator and parallel:
-        if ("Makefiles" in generator or "Ninja" in generator) and "NMake" not in generator:
-            args.append("-j%i" % cpu_count(conanfile.output))
-        elif "Visual Studio" in generator and compiler_version and Version(compiler_version) >= "10":
-            # Parallel for building projects in the solution
-            args.append("/m:%i" % cpu_count(output=conanfile.output))
-
-    if generator and "Visual Studio" in generator:
+    if not generator:
+        return args
+
+    # Arguments related to parallel
+    if parallel:
+        if "Makefiles" in generator and "NMake" not in generator:
+            njobs = make_jobs_cmd_line_arg(conanfile)
+            if njobs:
+                args.append(njobs)
+
+        if "Ninja" in generator and "NMake" not in generator:
+            njobs = ninja_jobs_cmd_line_arg(conanfile)
+            if njobs:
+                args.append(njobs)
+
+        if "Visual Studio" in generator:
+            max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(conanfile)
+            if max_cpu_count:
+                args.append(max_cpu_count)
+
+    # Arguments for verbosity
+    if "Visual Studio" in generator:
         verbosity = msbuild_verbosity_cmd_line_arg(conanfile)
         if verbosity:
             args.append(verbosity)

conan/tools/gnu/make.py

@@ -3,11 +3,19 @@
 from collections import OrderedDict
 
 from jinja2 import Template
+
 from conans.client.build.compiler_flags import build_type_define, libcxx_define
 from conans.client.tools.oss import detected_architecture, detected_os, get_build_os_arch
 from conans.util.files import save
 
 
+def make_jobs_cmd_line_arg(conanfile):
+    njobs = conanfile.conf["tools.gnu.make"].jobs or \
+            conanfile.conf["tools.build"].processes
+    if njobs:
+        return "-j{}".format(njobs)
+
+
 class MakeToolchain(object):
     filename = "conan_toolchain.mak"
 

conan/tools/meson/meson.py

@@ -2,7 +2,14 @@
 
 from conan.tools.meson import MesonToolchain
 from conan.tools.microsoft.visual import vcvars_command, vcvars_arch
-from conans.client.tools.oss import cross_building, cpu_count
+from conans.client.tools.oss import cross_building
+
+
+def ninja_jobs_cmd_line_arg(conanfile):
+    njobs = conanfile.conf["tools.ninja"].jobs or \
+            conanfile.conf["tools.build"].processes
+    if njobs:
+        return "-j{}".format(njobs)
 
 
 class Meson(object):
@@ -33,14 +40,17 @@ def configure(self, source_folder=None):
         if cross_building(self._conanfile):
             cmd += ' --cross-file "{}"'.format(MesonToolchain.cross_filename)
         else:
-            cmd += ' --native-file "{}"'. format(MesonToolchain.native_filename)
+            cmd += ' --native-file "{}"'.format(MesonToolchain.native_filename)
         cmd += ' "{}" "{}"'.format(self._build_dir, source)
         if self._conanfile.package_folder:
             cmd += ' -Dprefix="{}"'.format(self._conanfile.package_folder)
         self._run(cmd)
 
     def build(self, target=None):
-        cmd = 'meson compile -C "{}" -j {}'.format(self._build_dir, cpu_count())
+        cmd = 'meson compile -C "{}"'.format(self._build_dir)
+        njobs = ninja_jobs_cmd_line_arg(self._conanfile)
+        if njobs:
+            cmd += " {}".format(njobs)
         if target:
             cmd += " {}".format(target)
         self._run(cmd)

conan/tools/microsoft/msbuild.py

@@ -9,6 +9,13 @@ def msbuild_verbosity_cmd_line_arg(conanfile):
         return '/verbosity:{}'.format(verbosity)
 
 
+def msbuild_max_cpu_count_cmd_line_arg(conanfile):
+    max_cpu_count = conanfile.conf["tools.microsoft.msbuild"].max_cpu_count or \
+                    conanfile.conf["tools.build"].processes
+    if max_cpu_count:
+        return "/m:{}".format(max_cpu_count)
+
+
 class MSBuild(object):
     def __init__(self, conanfile):
         self._conanfile = conanfile
@@ -34,6 +41,10 @@ def command(self, sln):
         if verbosity:
             cmd += " {}".format(verbosity)
 
+        max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(self._conanfile)
+        if max_cpu_count:
+            cmd += " {}".format(max_cpu_count)
+
         return cmd
 
     def build(self, sln):

conans/test/unittests/tools/cmake/test_cmake_cmd_line_args.py

@@ -0,0 +1,54 @@
+import textwrap
+
+import pytest
+
+from conan.tools.cmake.cmake import _cmake_cmd_line_args
+from conans.model.conf import ConfDefinition
+from conans.test.utils.mocks import ConanFileMock
+
+
+@pytest.fixture
+def conanfile():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.gnu.make:jobs=40
+        tools.ninja:jobs=30
+        tools.microsoft.msbuild:max_cpu_count=20
+        tools.build:processes=10
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    return conanfile
+
+
+def test_no_generator(conanfile):
+    args = _cmake_cmd_line_args(conanfile, None, parallel=True)
+    assert not len(args)
+
+
+def test_makefiles(conanfile):
+    args = _cmake_cmd_line_args(conanfile, 'Unix Makefiles', parallel=True)
+    assert args == ['-j40']
+
+    args = _cmake_cmd_line_args(conanfile, 'Unix Makefiles', parallel=False)
+    assert not len(args)
+
+    args = _cmake_cmd_line_args(conanfile, 'NMake Makefiles', parallel=True)
+    assert not len(args)
+
+
+def test_ninja(conanfile):
+    args = _cmake_cmd_line_args(conanfile, 'Ninja', parallel=True)
+    assert ['-j30'] == args
+
+    args = _cmake_cmd_line_args(conanfile, 'Ninja', parallel=False)
+    assert not len(args)
+
+
+def test_visual_studio(conanfile):
+    args = _cmake_cmd_line_args(conanfile, 'Visual Studio 16 2019', parallel=True)
+    assert ['/m:20'] == args
+
+    args = _cmake_cmd_line_args(conanfile, 'Ninja', parallel=False)
+    assert not len(args)

conans/test/unittests/tools/gnu/test_make_jobs_cmd_line_arg.py

@@ -0,0 +1,53 @@
+import textwrap
+
+from conan.tools.gnu.make import make_jobs_cmd_line_arg
+from conans.model.conf import ConfDefinition
+from conans.test.utils.mocks import ConanFileMock
+
+
+def test_tools_build():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.build:processes=10
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = make_jobs_cmd_line_arg(conanfile)
+    assert njobs == "-j10"
+
+
+def test_tools_gnu_make():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.gnu.make:jobs=23
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = make_jobs_cmd_line_arg(conanfile)
+    assert njobs == "-j23"
+
+
+def test_both_values():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.gnu.make:jobs=23
+        tools.build:processes=10
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = make_jobs_cmd_line_arg(conanfile)
+    assert njobs == "-j23"
+
+
+def test_none():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = make_jobs_cmd_line_arg(conanfile)
+    assert njobs is None

conans/test/unittests/tools/meson/test_meson.py

@@ -0,0 +1,28 @@
+import textwrap
+
+from conan.tools.meson import Meson
+from conans.model.conf import ConfDefinition
+from conans.test.utils.mocks import ConanFileMock, MockSettings
+
+
+def test_meson_build():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.ninja:jobs=23
+        tools.build:processes=10
+    """))
+
+    settings = MockSettings({"build_type": "Release",
+                             "compiler": "gcc",
+                             "compiler.version": "7",
+                             "os": "Linux",
+                             "arch": "x86_64"})
+    conanfile = ConanFileMock()
+    conanfile.settings = settings
+    conanfile.display_name = 'test'
+    conanfile.conf = c.get_conanfile_conf(None)
+
+    meson = Meson(conanfile)
+    meson.build()
+
+    assert '-j23' in str(conanfile.command)

conans/test/unittests/tools/meson/test_ninja_jobs_cmd_line_arg.py

@@ -0,0 +1,53 @@
+import textwrap
+
+from conan.tools.meson.meson import ninja_jobs_cmd_line_arg
+from conans.model.conf import ConfDefinition
+from conans.test.utils.mocks import ConanFileMock
+
+
+def test_tools_build():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.build:processes=10
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = ninja_jobs_cmd_line_arg(conanfile)
+    assert njobs == "-j10"
+
+
+def test_tools_ning():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.ninja:jobs=23
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = ninja_jobs_cmd_line_arg(conanfile)
+    assert njobs == "-j23"
+
+
+def test_both_values():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.ninja:jobs=23
+        tools.build:processes=10
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = ninja_jobs_cmd_line_arg(conanfile)
+    assert njobs == "-j23"
+
+
+def test_none():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+    """))
+
+    conanfile = ConanFileMock()
+    conanfile.conf = c.get_conanfile_conf(None)
+    njobs = ninja_jobs_cmd_line_arg(conanfile)
+    assert njobs is None

conans/test/unittests/tools/microsoft/test_msbuild.py

@@ -0,0 +1,27 @@
+import textwrap
+
+from conan.tools.microsoft import MSBuild
+from conans.model.conf import ConfDefinition
+from conans.test.utils.mocks import ConanFileMock, MockSettings
+
+
+def test_msbuild_cpu_count():
+    c = ConfDefinition()
+    c.loads(textwrap.dedent("""\
+        tools.microsoft.msbuild:max_cpu_count=23
+        tools.build:processes=10
+    """))
+
+    settings = MockSettings({"build_type": "Release",
+                             "compiler": "gcc",
+                             "compiler.version": "7",
+                             "os": "Linux",
+                             "arch": "x86_64"})
+    conanfile = ConanFileMock()
+    conanfile.settings = settings
+    conanfile.conf = c.get_conanfile_conf(None)
+
+    msbuild = MSBuild(conanfile)
+    cmd = msbuild.command('project.sln')
+
+    assert '/m:23' in cmd