Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

More fine-grained control (using [conf]) for build parallelization #8665

Merged
merged 11 commits into from Mar 26, 2021
37 changes: 26 additions & 11 deletions conan/tools/cmake/cmake.py
Expand Up @@ -3,13 +3,15 @@

from conan.tools.cmake.base import CMakeToolchainBase
from conan.tools.cmake.utils import get_generator, is_multi_configuration
from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg
from conan.tools.gnu.make import make_jobs_cmd_line_arg
from conan.tools.meson.meson import ninja_jobs_cmd_line_arg
from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg, \
msbuild_max_cpu_count_cmd_line_arg
from conans.client import tools
from conans.client.build import join_arguments
from conans.client.tools.files import chdir
from conans.client.tools.oss import cpu_count, args_to_string
from conans.errors import ConanException
from conans.model.version import Version
from conans.util.conan_v2_mode import conan_v2_error
from conans.util.files import mkdir

Expand All @@ -23,15 +25,28 @@ def _validate_recipe(conanfile):

def _cmake_cmd_line_args(conanfile, generator, parallel):
args = []
compiler_version = conanfile.settings.get_safe("compiler.version")
if generator and parallel:
if ("Makefiles" in generator or "Ninja" in generator) and "NMake" not in generator:
args.append("-j%i" % cpu_count(conanfile.output))
elif "Visual Studio" in generator and compiler_version and Version(compiler_version) >= "10":
# Parallel for building projects in the solution
args.append("/m:%i" % cpu_count(output=conanfile.output))

if generator and "Visual Studio" in generator:
if not generator:
return args

# Arguments related to parallel
if parallel:
if "Makefiles" in generator and "NMake" not in generator:
njobs = make_jobs_cmd_line_arg(conanfile)
if njobs:
args.append(njobs)

if "Ninja" in generator and "NMake" not in generator:
njobs = ninja_jobs_cmd_line_arg(conanfile)
if njobs:
args.append(njobs)

if "Visual Studio" in generator:
max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(conanfile)
if max_cpu_count:
args.append(max_cpu_count)

# Arguments for verbosity
if "Visual Studio" in generator:
verbosity = msbuild_verbosity_cmd_line_arg(conanfile)
if verbosity:
args.append(verbosity)
Expand Down
8 changes: 8 additions & 0 deletions conan/tools/gnu/make.py
Expand Up @@ -3,11 +3,19 @@
from collections import OrderedDict

from jinja2 import Template

from conans.client.build.compiler_flags import build_type_define, libcxx_define
from conans.client.tools.oss import detected_architecture, detected_os, get_build_os_arch
from conans.util.files import save


def make_jobs_cmd_line_arg(conanfile):
njobs = conanfile.conf["tools.gnu.make"].jobs or \
conanfile.conf["tools.build"].processes
if njobs:
return "-j{}".format(njobs)


class MakeToolchain(object):
filename = "conan_toolchain.mak"

Expand Down
16 changes: 13 additions & 3 deletions conan/tools/meson/meson.py
Expand Up @@ -2,7 +2,14 @@

from conan.tools.meson import MesonToolchain
from conan.tools.microsoft.visual import vcvars_command, vcvars_arch
from conans.client.tools.oss import cross_building, cpu_count
from conans.client.tools.oss import cross_building


def ninja_jobs_cmd_line_arg(conanfile):
njobs = conanfile.conf["tools.ninja"].jobs or \
conanfile.conf["tools.build"].processes
if njobs:
return "-j{}".format(njobs)


class Meson(object):
Expand Down Expand Up @@ -33,14 +40,17 @@ def configure(self, source_folder=None):
if cross_building(self._conanfile):
cmd += ' --cross-file "{}"'.format(MesonToolchain.cross_filename)
else:
cmd += ' --native-file "{}"'. format(MesonToolchain.native_filename)
cmd += ' --native-file "{}"'.format(MesonToolchain.native_filename)
cmd += ' "{}" "{}"'.format(self._build_dir, source)
if self._conanfile.package_folder:
cmd += ' -Dprefix="{}"'.format(self._conanfile.package_folder)
self._run(cmd)

def build(self, target=None):
cmd = 'meson compile -C "{}" -j {}'.format(self._build_dir, cpu_count())
cmd = 'meson compile -C "{}"'.format(self._build_dir)
njobs = ninja_jobs_cmd_line_arg(self._conanfile)
if njobs:
cmd += " {}".format(njobs)
if target:
cmd += " {}".format(target)
self._run(cmd)
Expand Down
11 changes: 11 additions & 0 deletions conan/tools/microsoft/msbuild.py
Expand Up @@ -9,6 +9,13 @@ def msbuild_verbosity_cmd_line_arg(conanfile):
return '/verbosity:{}'.format(verbosity)


def msbuild_max_cpu_count_cmd_line_arg(conanfile):
max_cpu_count = conanfile.conf["tools.microsoft.msbuild"].max_cpu_count or \
conanfile.conf["tools.build"].processes
if max_cpu_count:
return "/m:{}".format(max_cpu_count)


class MSBuild(object):
def __init__(self, conanfile):
self._conanfile = conanfile
Expand All @@ -34,6 +41,10 @@ def command(self, sln):
if verbosity:
cmd += " {}".format(verbosity)

max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(self._conanfile)
if max_cpu_count:
cmd += " {}".format(max_cpu_count)

return cmd

def build(self, sln):
Expand Down
54 changes: 54 additions & 0 deletions conans/test/unittests/tools/cmake/test_cmake_cmd_line_args.py
@@ -0,0 +1,54 @@
import textwrap

import pytest

from conan.tools.cmake.cmake import _cmake_cmd_line_args
from conans.model.conf import ConfDefinition
from conans.test.utils.mocks import ConanFileMock


@pytest.fixture
def conanfile():
c = ConfDefinition()
c.loads(textwrap.dedent("""\
tools.gnu.make:jobs=40
tools.ninja:jobs=30
tools.microsoft.msbuild:max_cpu_count=20
tools.build:processes=10
"""))

conanfile = ConanFileMock()
conanfile.conf = c.get_conanfile_conf(None)
return conanfile


def test_no_generator(conanfile):
args = _cmake_cmd_line_args(conanfile, None, parallel=True)
assert not len(args)


def test_makefiles(conanfile):
args = _cmake_cmd_line_args(conanfile, 'Unix Makefiles', parallel=True)
assert args == ['-j40']

args = _cmake_cmd_line_args(conanfile, 'Unix Makefiles', parallel=False)
assert not len(args)

args = _cmake_cmd_line_args(conanfile, 'NMake Makefiles', parallel=True)
assert not len(args)


def test_ninja(conanfile):
args = _cmake_cmd_line_args(conanfile, 'Ninja', parallel=True)
assert ['-j30'] == args

args = _cmake_cmd_line_args(conanfile, 'Ninja', parallel=False)
assert not len(args)


def test_visual_studio(conanfile):
args = _cmake_cmd_line_args(conanfile, 'Visual Studio 16 2019', parallel=True)
assert ['/m:20'] == args

args = _cmake_cmd_line_args(conanfile, 'Ninja', parallel=False)
assert not len(args)
Empty file.
53 changes: 53 additions & 0 deletions conans/test/unittests/tools/gnu/test_make_jobs_cmd_line_arg.py
@@ -0,0 +1,53 @@
import textwrap

from conan.tools.gnu.make import make_jobs_cmd_line_arg
from conans.model.conf import ConfDefinition
from conans.test.utils.mocks import ConanFileMock


def test_tools_build():
c = ConfDefinition()
c.loads(textwrap.dedent("""\
tools.build:processes=10
"""))

conanfile = ConanFileMock()
conanfile.conf = c.get_conanfile_conf(None)
njobs = make_jobs_cmd_line_arg(conanfile)
assert njobs == "-j10"


def test_tools_gnu_make():
c = ConfDefinition()
c.loads(textwrap.dedent("""\
tools.gnu.make:jobs=23
"""))

conanfile = ConanFileMock()
conanfile.conf = c.get_conanfile_conf(None)
njobs = make_jobs_cmd_line_arg(conanfile)
assert njobs == "-j23"


def test_both_values():
c = ConfDefinition()
c.loads(textwrap.dedent("""\
tools.gnu.make:jobs=23
tools.build:processes=10
"""))

conanfile = ConanFileMock()
conanfile.conf = c.get_conanfile_conf(None)
njobs = make_jobs_cmd_line_arg(conanfile)
assert njobs == "-j23"


def test_none():
c = ConfDefinition()
c.loads(textwrap.dedent("""\
"""))

conanfile = ConanFileMock()
conanfile.conf = c.get_conanfile_conf(None)
njobs = make_jobs_cmd_line_arg(conanfile)
assert njobs is None
Empty file.
28 changes: 28 additions & 0 deletions conans/test/unittests/tools/meson/test_meson.py
@@ -0,0 +1,28 @@
import textwrap

from conan.tools.meson import Meson
from conans.model.conf import ConfDefinition
from conans.test.utils.mocks import ConanFileMock, MockSettings


def test_meson_build():
c = ConfDefinition()
c.loads(textwrap.dedent("""\
tools.ninja:jobs=23
tools.build:processes=10
"""))

settings = MockSettings({"build_type": "Release",
"compiler": "gcc",
"compiler.version": "7",
"os": "Linux",
"arch": "x86_64"})
conanfile = ConanFileMock()
conanfile.settings = settings
conanfile.display_name = 'test'
conanfile.conf = c.get_conanfile_conf(None)

meson = Meson(conanfile)
meson.build()

assert '-j23' in str(conanfile.command)
53 changes: 53 additions & 0 deletions conans/test/unittests/tools/meson/test_ninja_jobs_cmd_line_arg.py
@@ -0,0 +1,53 @@
import textwrap

from conan.tools.meson.meson import ninja_jobs_cmd_line_arg
from conans.model.conf import ConfDefinition
from conans.test.utils.mocks import ConanFileMock


def test_tools_build():
c = ConfDefinition()
c.loads(textwrap.dedent("""\
tools.build:processes=10
"""))

conanfile = ConanFileMock()
conanfile.conf = c.get_conanfile_conf(None)
njobs = ninja_jobs_cmd_line_arg(conanfile)
assert njobs == "-j10"


def test_tools_ning():
c = ConfDefinition()
c.loads(textwrap.dedent("""\
tools.ninja:jobs=23
"""))

conanfile = ConanFileMock()
conanfile.conf = c.get_conanfile_conf(None)
njobs = ninja_jobs_cmd_line_arg(conanfile)
assert njobs == "-j23"


def test_both_values():
c = ConfDefinition()
c.loads(textwrap.dedent("""\
tools.ninja:jobs=23
tools.build:processes=10
"""))

conanfile = ConanFileMock()
conanfile.conf = c.get_conanfile_conf(None)
njobs = ninja_jobs_cmd_line_arg(conanfile)
assert njobs == "-j23"


def test_none():
c = ConfDefinition()
c.loads(textwrap.dedent("""\
"""))

conanfile = ConanFileMock()
conanfile.conf = c.get_conanfile_conf(None)
njobs = ninja_jobs_cmd_line_arg(conanfile)
assert njobs is None
Empty file.
27 changes: 27 additions & 0 deletions conans/test/unittests/tools/microsoft/test_msbuild.py
@@ -0,0 +1,27 @@
import textwrap

from conan.tools.microsoft import MSBuild
from conans.model.conf import ConfDefinition
from conans.test.utils.mocks import ConanFileMock, MockSettings


def test_msbuild_cpu_count():
c = ConfDefinition()
c.loads(textwrap.dedent("""\
tools.microsoft.msbuild:max_cpu_count=23
tools.build:processes=10
"""))

settings = MockSettings({"build_type": "Release",
"compiler": "gcc",
"compiler.version": "7",
"os": "Linux",
"arch": "x86_64"})
conanfile = ConanFileMock()
conanfile.settings = settings
conanfile.conf = c.get_conanfile_conf(None)

msbuild = MSBuild(conanfile)
cmd = msbuild.command('project.sln')

assert '/m:23' in cmd