[develop2] Merge develop (#8717)

* Added existing tests to CMakeDeps and fixed some things about it (#8697)

* Added existing tests to CMakeDeps and fixed some things about it

* Review and fixed name of data file

* Removed wrong comment and fixed test by declaring filenames only for CMakeDeps

* new makefile_gen (#8705)

* new makefile_gen

* pytest mark reason

* Fix remotes not loaded for conan alias command (#8704)

* Fix remotes not loaded for alias command

* Add test

* adding diamond visits and avoid repetitions (#8701)

* Start using ConanFileDependencies in msbuilddeps  (#8706)

* start using ConanFileDependencies in MSBuildDeps generator

* full usage of ConanFileDependencies

* fix intel imports

* More fine-grained control (using [conf]) for build parallelization (#8665)

* first shot

* format the strings

* package_id change due to msbuild property name change

* do not fallback to cpu_count

* remove parallel from constructor

* remove change not needed

* add unittesting

* check once for parallel

* Update conans/test/unittests/tools/microsoft/test_msbuild.py

Co-authored-by: James <james@conan.io>

Co-authored-by: James <james@conan.io>

Co-authored-by: Luis Martinez de Bartolome Izquierdo <lasote@gmail.com>
Co-authored-by: Joe <8194899+cobalt77@users.noreply.github.com>
Co-authored-by: Javier G. Sogo <jgsogo@gmail.com>

conan/tools/cmake/cmake.py

@@ -3,13 +3,15 @@
 
 from conan.tools.cmake.base import CMakeToolchainBase
 from conan.tools.cmake.utils import get_generator, is_multi_configuration
-from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg
+from conan.tools.gnu.make import make_jobs_cmd_line_arg
+from conan.tools.meson.meson import ninja_jobs_cmd_line_arg
+from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg, \
+    msbuild_max_cpu_count_cmd_line_arg
 from conans.client import tools
 from conans.client.build import join_arguments
 from conans.client.tools.files import chdir
 from conans.client.tools.oss import cpu_count, args_to_string
 from conans.errors import ConanException
-from conans.model.version import Version
 from conans.util.conan_v2_mode import conan_v2_error
 from conans.util.files import mkdir
 
@@ -23,15 +25,28 @@ def _validate_recipe(conanfile):
 
 def _cmake_cmd_line_args(conanfile, generator, parallel):
     args = []
-    compiler_version = conanfile.settings.get_safe("compiler.version")
-    if generator and parallel:
-        if ("Makefiles" in generator or "Ninja" in generator) and "NMake" not in generator:
-            args.append("-j%i" % cpu_count(conanfile.output))
-        elif "Visual Studio" in generator and compiler_version and Version(compiler_version) >= "10":
-            # Parallel for building projects in the solution
-            args.append("/m:%i" % cpu_count(output=conanfile.output))
-
-    if generator and "Visual Studio" in generator:
+    if not generator:
+        return args
+
+    # Arguments related to parallel
+    if parallel:
+        if "Makefiles" in generator and "NMake" not in generator:
+            njobs = make_jobs_cmd_line_arg(conanfile)
+            if njobs:
+                args.append(njobs)
+
+        if "Ninja" in generator and "NMake" not in generator:
+            njobs = ninja_jobs_cmd_line_arg(conanfile)
+            if njobs:
+                args.append(njobs)
+
+        if "Visual Studio" in generator:
+            max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(conanfile)
+            if max_cpu_count:
+                args.append(max_cpu_count)
+
+    # Arguments for verbosity
+    if "Visual Studio" in generator:
         verbosity = msbuild_verbosity_cmd_line_arg(conanfile)
         if verbosity:
             args.append(verbosity)

conan/tools/cmake/cmakedeps.py

@@ -267,7 +267,7 @@ class CMakeDeps(object):
 
         # Load the debug and release library finders
         get_filename_component(_DIR "${{CMAKE_CURRENT_LIST_FILE}}" PATH)
-        file(GLOB DATA_FILES "${{_DIR}}/{filename}-*-*-data.cmake")
+        file(GLOB DATA_FILES "${{_DIR}}/{filename}-*-data.cmake")
 
         foreach(f ${{DATA_FILES}})
             include(${{f}})
@@ -419,7 +419,7 @@ class CMakeDeps(object):
 
         # Load the debug and release variables
         get_filename_component(_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
-        file(GLOB DATA_FILES "${_DIR}/{{ pkg_filename }}-*-*-data.cmake")
+        file(GLOB DATA_FILES "${_DIR}/{{ pkg_filename }}-*-data.cmake")
 
         foreach(f ${DATA_FILES})
             include(${f})
@@ -529,7 +529,7 @@ class CMakeDeps(object):
 
     def __init__(self, conanfile):
         self._conanfile = conanfile
-        self.arch = str(self._conanfile.settings.arch)
+        self.arch = self._conanfile.settings.get_safe("arch")
         self.configuration = str(self._conanfile.settings.build_type)
         self.configurations = [v for v in conanfile.settings.build_type.values_range if v != "None"]
         # FIXME: Ugly way to define the output path
@@ -626,6 +626,13 @@ def generate(self):
             generator_file = os.path.join(self.output_path, generator_file)
             save(generator_file, content)
 
+    def _data_filename(self, pkg_filename):
+        data_fname = "{}-{}".format(pkg_filename, self.configuration.lower())
+        if self.arch:
+            data_fname += "-{}".format(self.arch)
+        data_fname += "-data.cmake"
+        return data_fname
+
     @property
     def content(self):
         ret = {}
@@ -659,12 +666,11 @@ def content(self):
                 # If any config matches the build_type one, add it to the cpp_info
                 dep_cpp_info = extend(cpp_info, build_type.lower())
                 deps = DepsCppCmake(dep_cpp_info, self.name)
+
                 variables = {
-                    "{name}-{build_type}-{arch}-data.cmake".format(name=pkg_filename,
-                                                                   build_type=self.configuration.lower(),
-                                                                   arch=self.arch):
-                    variables_template.format(name=pkg_findname, deps=deps,
-                                              build_type_suffix=build_type_suffix)
+                   self._data_filename(pkg_filename):
+                       variables_template.format(name=pkg_findname, deps=deps,
+                                                 build_type_suffix=build_type_suffix)
                              }
                 dynamic_variables = {
                     "{}Target-{}.cmake".format(pkg_filename, self.configuration.lower()):
@@ -693,10 +699,10 @@ def content(self):
                                                              build_type_suffix=build_type_suffix,
                                                              deps_names=deps_names)
                 variables = {
-                    "{}-{}-{}-data.cmake".format(pkg_filename, build_type.lower(), self.arch):
-                    self.components_variables_tpl.render(
-                        pkg_name=pkg_findname, global_variables=global_variables,
-                        pkg_components=pkg_components, build_type=build_type, components=components)
+                    self._data_filename(pkg_filename):
+                        self.components_variables_tpl.render(
+                         pkg_name=pkg_findname, global_variables=global_variables,
+                         pkg_components=pkg_components, build_type=build_type, components=components)
                 }
                 ret.update(variables)
                 global_dynamic_variables = dynamic_variables_template.format(name=pkg_findname,

conan/tools/env/environment.py

@@ -198,6 +198,8 @@ def save_sh(self, filename, generate_deactivate=False, pathsep=os.pathsep):
 
     def compose(self, other):
         """
+        self has precedence, the "other" will add/append if possible and not conflicting, but
+        self mandates what to do
         :type other: Environment
         """
         for k, v in other._values.items():
@@ -206,12 +208,12 @@ def compose(self, other):
                 self._values[k] = v
             else:
                 try:
-                    index = v.index(_EnvVarPlaceHolder)
-                except ValueError:  # The other doesn't have placeholder, overwrites
-                    self._values[k] = v
+                    index = existing.index(_EnvVarPlaceHolder)
+                except ValueError:  # It doesn't have placeholder
+                    pass
                 else:
-                    new_value = v[:]  # do a copy
-                    new_value[index:index + 1] = existing  # replace the placeholder
+                    new_value = existing[:]  # do a copy
+                    new_value[index:index + 1] = v  # replace the placeholder
                     # Trim front and back separators
                     val = new_value[0]
                     if isinstance(val, _Sep) or val is _PathSep:
@@ -233,11 +235,12 @@ def __repr__(self):
     def get_env(self, ref):
         """ computes package-specific Environment
         it is only called when conanfile.buildenv is called
+        the last one found in the profile file has top priority
         """
         result = Environment()
         for pattern, env in self._environments.items():
             if pattern is None or fnmatch.fnmatch(str(ref), pattern):
-                result = result.compose(env)
+                result = env.compose(result)
         return result
 
     def compose(self, other):
@@ -247,7 +250,7 @@ def compose(self, other):
         for pattern, environment in other._environments.items():
             existing = self._environments.get(pattern)
             if existing is not None:
-                self._environments[pattern] = existing.compose(environment)
+                self._environments[pattern] = environment.compose(existing)
             else:
                 self._environments[pattern] = environment
 
@@ -283,7 +286,7 @@ def loads(text):
                 if existing is None:
                     result._environments[pattern] = env
                 else:
-                    result._environments[pattern] = existing.compose(env)
+                    result._environments[pattern] = env.compose(existing)
                 break
             else:
                 raise ConanException("Bad env defintion: {}".format(line))

conan/tools/env/virtualenv.py

@@ -18,32 +18,42 @@ def build_environment(self):
         of build_requires defining information for consumers
         """
         build_env = Environment()
+        # Top priority: profile
+        profile_env = self._conanfile.buildenv
+        build_env.compose(profile_env)
+
         # First visit the direct build_requires
+        transitive_requires = []
         for build_require in self._conanfile.dependencies.build_requires:
-            # Lower priority, the runenv of all transitive "requires" of the build requires
-            for require in build_require.dependencies.requires:
-                build_env.compose(self._collect_transitive_runenv(require))
-            # Second, the implicit self information in build_require.cpp_info
-            build_env.compose(self._runenv_from_cpp_info(build_require.cpp_info))
-            # Finally, higher priority, explicit buildenv_info
+            # higher priority, explicit buildenv_info
             if build_require.buildenv_info:
                 build_env.compose(build_require.buildenv_info)
+            # Second, the implicit self information in build_require.cpp_info
+            build_env.compose(self._runenv_from_cpp_info(build_require.cpp_info))
+            # Lower priority, the runenv of all transitive "requires" of the build requires
+            for require in build_require.dependencies.requires:
+                if require not in transitive_requires:
+                    transitive_requires.append(require)
+
+        self._apply_transitive_runenv(transitive_requires, build_env)
 
         # Requires in host context can also bring some direct buildenv_info
-        def _collect_transitive_buildenv(d):
-            r = Environment()
-            for child in d.dependencies.requires:
-                r.compose(_collect_transitive_buildenv(child))
-            # Then the explicit self
-            if d.buildenv_info:
-                r.compose(d.buildenv_info)
-            return r
-        for require in self._conanfile.dependencies.requires:
-            build_env.compose(_collect_transitive_buildenv(require))
-
-        # The profile environment has precedence, applied last
-        profile_env = self._conanfile.buildenv
-        build_env.compose(profile_env)
+        def _apply_transitive_buildenv(reqs, env):
+            all_requires = []
+            while reqs:
+                new_requires = []
+                for r in reqs:
+                    # The explicit has more priority
+                    if r.buildenv_info:
+                        env.compose(r.buildenv_info)
+                    for transitive in r.dependencies.requires:
+                        # Avoid duplication/repetitions
+                        if transitive not in new_requires and transitive not in all_requires:
+                            new_requires.append(transitive)
+                reqs = new_requires
+
+        _apply_transitive_buildenv(self._conanfile.dependencies.requires, build_env)
+
         return build_env
 
     @staticmethod
@@ -63,33 +73,40 @@ def _runenv_from_cpp_info(cpp_info):
             dyn_runenv.prepend_path("DYLD_FRAMEWORK_PATH", cpp_info.framework_paths)
         return dyn_runenv
 
-    def _collect_transitive_runenv(self, d):
-        r = Environment()
-        for child in d.dependencies.requires:
-            r.compose(self._collect_transitive_runenv(child))
-        # Apply "d" runenv, first the implicit
-        r.compose(self._runenv_from_cpp_info(d.cpp_info))
-        # Then the explicit
-        if d.runenv_info:
-            r.compose(d.runenv_info)
-        return r
+    def _apply_transitive_runenv(self, next_requires, runenv):
+        all_requires = []
+        while next_requires:
+            new_requires = []
+            for require in next_requires:
+                # The explicit has more priority
+                if require.runenv_info:
+                    runenv.compose(require.runenv_info)
+                # Then the implicit
+                runenv.compose(self._runenv_from_cpp_info(require.cpp_info))
+                all_requires.append(require)
+
+                for transitive in require.dependencies.requires:
+                    # Avoid duplication/repetitions
+                    if transitive not in new_requires and transitive not in all_requires:
+                        new_requires.append(transitive)
+            next_requires = new_requires
 
     def run_environment(self):
         """ collects the runtime information from dependencies. For normal libraries should be
         very occasional
         """
         runenv = Environment()
+        # FIXME: Missing profile info
+
+        # Visitor, breadth-first
+        self._apply_transitive_runenv(self._conanfile.dependencies.requires, runenv)
         # At the moment we are adding "test-requires" (build_requires in host context)
         # to the "runenv", but this will be investigated
-        for build_require in self._conanfile.dependencies.build_requires:
-            if build_require.context == CONTEXT_HOST:
-                runenv.compose(self._collect_transitive_runenv(build_require))
-        for require in self._conanfile.dependencies.requires:
-            runenv.compose(self._collect_transitive_runenv(require))
+        host_build_requires = [br for br in self._conanfile.dependencies.build_requires
+                               if br.context == CONTEXT_HOST]
+        self._apply_transitive_runenv(host_build_requires, runenv)
 
-        # FIXME: Missing profile info
-        result = runenv
-        return result
+        return runenv
 
     def generate(self):
         build_env = self.build_environment()

conan/tools/gnu/make.py

@@ -3,11 +3,19 @@
 from collections import OrderedDict
 
 from jinja2 import Template
+
 from conans.client.build.compiler_flags import build_type_define, libcxx_define
 from conans.client.tools.oss import detected_architecture, detected_os, get_build_os_arch
 from conans.util.files import save
 
 
+def make_jobs_cmd_line_arg(conanfile):
+    njobs = conanfile.conf["tools.gnu.make"].jobs or \
+            conanfile.conf["tools.build"].processes
+    if njobs:
+        return "-j{}".format(njobs)
+
+
 class MakeToolchain(object):
     filename = "conan_toolchain.mak"
 

conan/tools/meson/meson.py

@@ -2,7 +2,14 @@
 
 from conan.tools.meson import MesonToolchain
 from conan.tools.microsoft.visual import vcvars_command, vcvars_arch
-from conans.client.tools.oss import cross_building, cpu_count
+from conans.client.tools.oss import cross_building
+
+
+def ninja_jobs_cmd_line_arg(conanfile):
+    njobs = conanfile.conf["tools.ninja"].jobs or \
+            conanfile.conf["tools.build"].processes
+    if njobs:
+        return "-j{}".format(njobs)
 
 
 class Meson(object):
@@ -33,14 +40,17 @@ def configure(self, source_folder=None):
         if cross_building(self._conanfile):
             cmd += ' --cross-file "{}"'.format(MesonToolchain.cross_filename)
         else:
-            cmd += ' --native-file "{}"'. format(MesonToolchain.native_filename)
+            cmd += ' --native-file "{}"'.format(MesonToolchain.native_filename)
         cmd += ' "{}" "{}"'.format(self._build_dir, source)
         if self._conanfile.package_folder:
             cmd += ' -Dprefix="{}"'.format(self._conanfile.package_folder)
         self._run(cmd)
 
     def build(self, target=None):
-        cmd = 'meson compile -C "{}" -j {}'.format(self._build_dir, cpu_count())
+        cmd = 'meson compile -C "{}"'.format(self._build_dir)
+        njobs = ninja_jobs_cmd_line_arg(self._conanfile)
+        if njobs:
+            cmd += " {}".format(njobs)
         if target:
             cmd += " {}".format(target)
         self._run(cmd)

conan/tools/microsoft/msbuild.py

@@ -9,6 +9,13 @@ def msbuild_verbosity_cmd_line_arg(conanfile):
         return '/verbosity:{}'.format(verbosity)
 
 
+def msbuild_max_cpu_count_cmd_line_arg(conanfile):
+    max_cpu_count = conanfile.conf["tools.microsoft.msbuild"].max_cpu_count or \
+                    conanfile.conf["tools.build"].processes
+    if max_cpu_count:
+        return "/m:{}".format(max_cpu_count)
+
+
 class MSBuild(object):
     def __init__(self, conanfile):
         self._conanfile = conanfile
@@ -34,6 +41,10 @@ def command(self, sln):
         if verbosity:
             cmd += " {}".format(verbosity)
 
+        max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(self._conanfile)
+        if max_cpu_count:
+            cmd += " {}".format(max_cpu_count)
+
         return cmd
 
     def build(self, sln):