From 75ff5126250430bda75ce857ee63aa504558d993 Mon Sep 17 00:00:00 2001 From: memsharded Date: Sun, 21 Feb 2021 21:05:03 +0100 Subject: [PATCH 01/28] proposal --- .gitignore | 1 - conan/tools/env/__init__.py | 2 + conan/tools/env/environment.py | 230 ++++++++++++++++++ conan/tools/env/virtualenv.py | 81 ++++++ conans/client/generators/__init__.py | 8 +- conans/client/graph/graph_builder.py | 3 +- conans/client/loader.py | 6 +- conans/client/profile_loader.py | 10 +- conans/model/conan_file.py | 17 +- conans/model/conanfile_dependencies.py | 38 +++ conans/model/conanfile_interface.py | 22 ++ conans/model/profile.py | 3 + conans/test/functional/toolchains/test_env.py | 112 +++++++++ conans/test/unittests/tools/env/__init__.py | 0 conans/test/unittests/tools/env/test_env.py | 212 ++++++++++++++++ 15 files changed, 735 insertions(+), 10 deletions(-) create mode 100644 conan/tools/env/__init__.py create mode 100644 conan/tools/env/environment.py create mode 100644 conan/tools/env/virtualenv.py create mode 100644 conans/model/conanfile_dependencies.py create mode 100644 conans/model/conanfile_interface.py create mode 100644 conans/test/functional/toolchains/test_env.py create mode 100644 conans/test/unittests/tools/env/__init__.py create mode 100644 conans/test/unittests/tools/env/test_env.py diff --git a/.gitignore b/.gitignore index 2e4f6277c3e..4a27bb1ea69 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ __pycache__/ # Distribution / packaging .Python -env/ build/ develop-eggs/ dist/ diff --git a/conan/tools/env/__init__.py b/conan/tools/env/__init__.py new file mode 100644 index 00000000000..0eed8d1a0f5 --- /dev/null +++ b/conan/tools/env/__init__.py @@ -0,0 +1,2 @@ +from conan.tools.env.environment import Environment +from conan.tools.env.virtualenv import VirtualEnv diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py new file mode 100644 index 00000000000..83b4146356a --- /dev/null +++ b/conan/tools/env/environment.py @@ -0,0 +1,230 @@ +import fnmatch +import os +import textwrap +from collections import OrderedDict + +from conans.errors import ConanException +from conans.util.files import save + +_ENV_VAR_PLACEHOLDER = "$PREVIOUS_ENV_VAR_VALUE%" +_PATHSEP = "$CONAN_PATHSEP%" + + +class Environment: + def __init__(self): + # TODO: Maybe we need to pass conanfile to get the [conf] + # It being ordered allows for Windows case-insensitive composition + self._values = OrderedDict() # {var_name: [] of values, including separators} + + def vars(self): + return list(self._values.keys()) + + def value(self, name, placeholder="{}", pathsep=os.pathsep): + return self._format_value(name, self._values[name], placeholder, pathsep) + + @staticmethod + def _format_value(name, varvalues, placeholder, pathsep): + values = [] + for v in varvalues: + if v == _ENV_VAR_PLACEHOLDER: + values.append(placeholder.format(name=name)) + elif v == _PATHSEP: + values.append(pathsep) + else: + values.append(v) + return "".join(values) + + @staticmethod + def _list_value(value, separator): + if isinstance(value, list): + result = [] + for v in value[:-1]: + result.append(v) + result.append(separator) + result.extend(value[-1:]) + return result + else: + return [value] + + def define(self, name, value, separator=" "): + value = self._list_value(value, separator) + self._values[name] = value + + def define_path(self, name, value): + self.define(name, value, _PATHSEP) + + def unset(self, name): + """ + clears the variable, equivalent to a unset or set XXX= + """ + self._values[name] = [] + + def append(self, name, value, separator=" "): + value = self._list_value(value, separator) + self._values[name] = [_ENV_VAR_PLACEHOLDER] + [separator] + value + + def append_path(self, name, value): + self.append(name, value, _PATHSEP) + + def prepend(self, name, value, separator=" "): + value = self._list_value(value, separator) + self._values[name] = value + [separator] + [_ENV_VAR_PLACEHOLDER] + + def prepend_path(self, name, value): + self.prepend(name, value, _PATHSEP) + + def save_bat(self, filename, generate_deactivate=True, pathsep=os.pathsep): + deactivate = textwrap.dedent("""\ + echo Capturing current environment in deactivate_{filename} + setlocal + echo @echo off > "deactivate_{filename}" + echo echo Restoring environment >> "deactivate_{filename}" + for %%v in ({vars}) do ( + set foundenvvar= + for /f "delims== tokens=1,2" %%a in ('set') do ( + if "%%a" == "%%v" ( + echo set %%a=%%b>> "deactivate_{filename}" + set foundenvvar=1 + ) + ) + if not defined foundenvvar ( + echo set %%v=>> "deactivate_{filename}" + ) + ) + endlocal + + """).format(filename=filename, vars=" ".join(self._values.keys())) + capture = textwrap.dedent("""\ + @echo off + {deactivate} + echo Configuring environment variables + """).format(deactivate=deactivate if generate_deactivate else "") + result = [capture] + for varname, varvalues in self._values.items(): + value = self._format_value(varname, varvalues, "%{name}%", pathsep) + result.append('set {}={}'.format(varname, value)) + + content = "\n".join(result) + save(filename, content) + + def save_ps1(self, filename, generate_deactivate=True, pathsep=os.pathsep): + # FIXME: This is broken and doesnt work + deactivate = "" + capture = textwrap.dedent("""\ + {deactivate} + """).format(deactivate=deactivate if generate_deactivate else "") + result = [capture] + for varname, varvalues in self._values.items(): + value = self._format_value(varname, varvalues, "$env:{name}", pathsep) + result.append('Write-Output "Error: whatever message {}"'.format(varname)) + result.append('$env:{}={}'.format(varname, value)) + + content = "\n".join(result) + save(filename, content) + + def save_sh(self, filename, pathsep=os.pathsep): + capture = textwrap.dedent("""\ + echo Capturing current environment in deactivate_{filename} + echo echo Restoring variables >> deactivate_{filename} + for v in {vars} + do + value=${{!v}} + if [ -n "$value" ] + then + echo export "$v=$value" >> deactivate_{filename} + else + echo unset $v >> deactivate_{filename} + fi + done + echo Configuring environment variables + """.format(filename=filename, vars=" ".join(self._values.keys()))) + result = [capture] + for varname, varvalues in self._values.items(): + value = self._format_value(varname, varvalues, "${name}", pathsep) + if value: + result.append('export {}="{}"'.format(varname, value)) + else: + result.append('unset {}'.format(varname)) + + content = "\n".join(result) + save(filename, content) + + def compose(self, other): + """ + :type other: Environment + """ + for k, v in other._values.items(): + existing = self._values.get(k) + if existing is None: + self._values[k] = v + else: + try: + index = v.index(_ENV_VAR_PLACEHOLDER) + except ValueError: # The other doesn't have placeholder, overwrites + self._values[k] = v + else: + new_value = v[:] # do a copy + new_value[index:index + 1] = existing # replace the placeholder + self._values[k] = new_value + + return self + + +class ProfileEnvironment: + def __init__(self): + self._environments = OrderedDict() + + def get_env(self, ref): + # TODO: Maybe we want to make this lazy, so this is not evaluated for every package + result = Environment() + for pattern, env in self._environments.items(): + if pattern is None or fnmatch.fnmatch(str(ref), pattern): + env = self._environments[pattern] + result = result.compose(env) + return result + + def compose(self, other): + """ + :type other: ProfileEnvironment + """ + for pattern, environment in other._environments.items(): + existing = self._environments.get(pattern) + if existing is not None: + self._environments[pattern] = existing.compose(environment) + else: + self._environments[pattern] = environment + + def loads(self, text): + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + for op, method in (("+=", "append"), ("=+", "prepend"), + ("=!", "unset"), ("=", "define")): + tokens = line.split(op, 1) + if len(tokens) != 2: + continue + pattern_name, value = tokens + pattern_name = pattern_name.split(":", 1) + if len(pattern_name) == 2: + pattern, name = pattern_name + else: + pattern, name = None, pattern_name[0] + + env = Environment() + if method == "unset": + env.unset(name) + else: + if value.startswith("(path)"): + value = value[6:] + method = method + "_path" + getattr(env, method)(name, value) + + existing = self._environments.get(pattern) + if existing is None: + self._environments[pattern] = env + else: + self._environments[pattern] = existing.compose(env) + break + else: + raise ConanException("Bad env defintion: {}".format(line)) diff --git a/conan/tools/env/virtualenv.py b/conan/tools/env/virtualenv.py new file mode 100644 index 00000000000..dd016aae28d --- /dev/null +++ b/conan/tools/env/virtualenv.py @@ -0,0 +1,81 @@ +import platform + +from conan.tools.env import Environment + + +class VirtualEnv: + """ captures the conanfile environment that is defined from its + dependencies, and also from profiles + """ + + def __init__(self, conanfile): + self._conanfile = conanfile + + def build_environment(self): + """ collects the buildtime information from dependencies. This is the typical use case + of build_requires defining information for consumers + """ + # Visit all dependencies + deps_env = Environment() + # TODO: The visitor of dependencies needs to be implemented correctly + for dep in self._conanfile.dependencies.all: + # environ_info is always "build" + dep_env = dep.buildenv_info + if dep_env is not None: + deps_env = deps_env.compose(dep_env) + + # The profile environment has precedence, applied last + profile_env = self._conanfile.buildenv + result = deps_env.compose(profile_env) + return result + + def autorun_environment(self): + """ automatically collects the runtime environment from 'cpp_info' from dependencies + By default is enabled and will be captured in 'runenv.xxx' but maybe can be disabled + by parameter or [conf] + """ + dyn_runenv = Environment() + # TODO: New cpp_info access + for dep in self._conanfile.dependencies.all: + cpp_info = dep.cpp_info + if cpp_info.exes: + dyn_runenv.append_path("PATH", cpp_info.bin_paths) + os_ = self._conanfile.settings.get_safe("os") + if cpp_info.lib_paths: + if os_ == "Linux": + dyn_runenv.append_path("LD_LIBRARY_PATH", cpp_info.lib_paths) + elif os_ == "Macos": + dyn_runenv.append_path("DYLD_LIBRARY_PATH", cpp_info.lib_paths) + if cpp_info.framework_paths: + dyn_runenv.append_path("DYLD_FRAMEWORK_PATH", cpp_info.framework_paths) + return dyn_runenv + + def run_environment(self): + """ collects the runtime information from dependencies. For normal libraries should be + very occasional + """ + # Visit all dependencies + deps_env = Environment() + for dep in self._conanfile.dependencies.all: + # run_environ_info is always "host" + dep_env = dep.runenv_info + if dep_env is not None: + deps_env = deps_env.compose(dep_env) + + autorun = self.autorun_environment() + deps_env = deps_env.compose(autorun) + # FIXME: Missing profile info + result = deps_env + return result + + def generate(self): + build_env = self.build_environment() + run_env = self.run_environment() + # FIXME: Use settings, not platform Not always defined :( + # os_ = self._conanfile.settings_build.get_safe("os") + if platform.system() == "Windows": + build_env.save_bat("buildenv.bat") + run_env.save_bat("runenv.bat") + else: + build_env.save_sh("buildenv.sh") + run_env.save_sh("runenv.sh") diff --git a/conans/client/generators/__init__.py b/conans/client/generators/__init__.py index 3f0c96a6281..6f4f3d06a4f 100644 --- a/conans/client/generators/__init__.py +++ b/conans/client/generators/__init__.py @@ -66,7 +66,8 @@ def __init__(self): "deploy": DeployGenerator, "markdown": MarkdownGenerator} self._new_generators = ["CMakeToolchain", "CMakeDeps", "MakeToolchain", "MSBuildToolchain", - "MesonToolchain", "MSBuildDeps", "QbsToolchain", "msbuild"] + "MesonToolchain", "MSBuildDeps", "QbsToolchain", "msbuild", + "VirtualEnv"] def add(self, name, generator_class, custom=False): if name not in self._generators or custom: @@ -110,6 +111,9 @@ def _new_generator(self, generator_name, output): elif generator_name == "QbsToolchain": from conan.tools.qbs.qbstoolchain import QbsToolchain return QbsToolchain + elif generator_name == "VirtualEnv": + from conan.tools.env.virtualenv import VirtualEnv + return VirtualEnv else: raise ConanException("Internal Conan error: Generator '{}' " "not commplete".format(generator_name)) @@ -195,5 +199,3 @@ def write_toolchain(conanfile, path, output): with chdir(path): with conanfile_exception_formatter(str(conanfile), "generate"): conanfile.generate() - - # TODO: Lets discuss what to do with the environment diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py index 026b12998a5..45b866a60b3 100644 --- a/conans/client/graph/graph_builder.py +++ b/conans/client/graph/graph_builder.py @@ -205,7 +205,7 @@ def _expand_require(self, require, node, graph, check_updates, update, remotes, remotes, profile_host, profile_build, graph_lock, context_switch=context_switch, populate_settings_target=populate_settings_target) - + node.conanfile.dependencies.add(new_node.conanfile) # The closure of a new node starts with just itself new_node.public_closure.add(new_node) new_node.transitive_closure[new_node.name] = new_node @@ -237,6 +237,7 @@ def _expand_require(self, require, node, graph, check_updates, update, remotes, node.transitive_closure[name] = n else: # a public node already exist with this name + node.conanfile.dependencies.add(previous.conanfile) self._resolve_cached_alias([require], graph) # As we are closing a diamond, there can be conflicts. This will raise if conflicts conflict = self._conflicting_references(previous, require.ref, node.ref) diff --git a/conans/client/loader.py b/conans/client/loader.py index 9851fb6e42e..f44d783b6ff 100644 --- a/conans/client/loader.py +++ b/conans/client/loader.py @@ -190,7 +190,7 @@ def _initialize_conanfile(conanfile, profile): if pkg_settings: tmp_settings.update_values(pkg_settings) - conanfile.initialize(tmp_settings, profile.env_values) + conanfile.initialize(tmp_settings, profile.env_values, profile.buildenv) conanfile.conf = profile.conf.get_conanfile_conf(ref_str) def load_consumer(self, conanfile_path, profile_host, name=None, version=None, user=None, @@ -259,7 +259,7 @@ def load_conanfile_txt(self, conan_txt_path, profile_host, ref=None): def _parse_conan_txt(self, contents, path, display_name, profile): conanfile = ConanFile(self._output, self._runner, display_name) - conanfile.initialize(Settings(), profile.env_values) + conanfile.initialize(Settings(), profile.env_values, profile.buildenv) conanfile.conf = profile.conf.get_conanfile_conf(None) # It is necessary to copy the settings, because the above is only a constraint of # conanfile settings, and a txt doesn't define settings. Necessary for generators, @@ -299,7 +299,7 @@ def load_virtual(self, references, profile_host, scope_options=True, # for the reference (keep compatibility) conanfile = ConanFile(self._output, self._runner, display_name="virtual") conanfile.initialize(profile_host.processed_settings.copy(), - profile_host.env_values) + profile_host.env_values, profile_host.buildenv) conanfile.conf = profile_host.conf.get_conanfile_conf(None) conanfile.settings = profile_host.processed_settings.copy_values() diff --git a/conans/client/profile_loader.py b/conans/client/profile_loader.py index 67cf0313dc6..a0f83fbca19 100644 --- a/conans/client/profile_loader.py +++ b/conans/client/profile_loader.py @@ -1,6 +1,7 @@ import os from collections import OrderedDict, defaultdict +from conan.tools.env.environment import ProfileEnvironment from conans.errors import ConanException, ConanV2Exception from conans.model.conf import ConfDefinition from conans.model.env_info import EnvValues, unquote @@ -149,7 +150,8 @@ def _load_profile(text, profile_path, default_folder): # Current profile before update with parents (but parent variables already applied) doc = ConfigParser(profile_parser.profile_text, - allowed_fields=["build_requires", "settings", "env", "options", "conf"]) + allowed_fields=["build_requires", "settings", "env", "options", "conf", + "buildenv"]) # Merge the inherited profile with the readed from current profile _apply_inner_profile(doc, inherited_profile) @@ -224,6 +226,12 @@ def get_package_name_value(item): new_prof.loads(doc.conf, profile=True) base_profile.conf.update_conf_definition(new_prof) + if doc.buildenv: + buildenv = ProfileEnvironment() + buildenv.loads(doc.buildenv) + # TODO: compose ProfileEnvironment + base_profile.buildenv = buildenv + def profile_from_args(profiles, settings, options, env, cwd, cache): """ Return a Profile object, as the result of merging a potentially existing Profile diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index 20b6a3c7e7e..c8959729b92 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -4,12 +4,14 @@ import six from six import string_types +from conan.tools.env import Environment from conans.client import tools from conans.client.output import ScopedOutput from conans.client.tools.env import environment_append, no_op, pythonpath from conans.client.tools.oss import OSInfo from conans.errors import ConanException, ConanInvalidConfiguration from conans.model.build_info import DepsCppInfo +from conans.model.conanfile_dependencies import ConanFileDependencies from conans.model.env_info import DepsEnvInfo from conans.model.layout import Layout from conans.model.options import Options, OptionsValues, PackageOptions @@ -145,8 +147,21 @@ def __init__(self, output, runner, display_name="", user=None, channel=None): self._conan_using_build_profile = False self.layout = Layout() + self.dependencies = ConanFileDependencies() + self.buildenv_info = Environment() + self.runenv_info = Environment() + self._conan_buildenv = None # The profile buildenv, will be assigned initialize() - def initialize(self, settings, env): + @property + def buildenv(self): + # Lazy computation of the package buildenv based on the profileone + if not isinstance(self._conan_buildenv, Environment): + ref_str = "{}/{}".format(self.name, self.version) + self._conan_buildenv = self._conan_buildenv.get_env(ref_str) + return self._conan_buildenv + + def initialize(self, settings, env, buildenv=None): + self._conan_buildenv = buildenv if isinstance(self.generators, str): self.generators = [self.generators] # User defined options diff --git a/conans/model/conanfile_dependencies.py b/conans/model/conanfile_dependencies.py new file mode 100644 index 00000000000..00fdc44d091 --- /dev/null +++ b/conans/model/conanfile_dependencies.py @@ -0,0 +1,38 @@ +from collections import OrderedDict + +from conans.model.conanfile_interface import ConanFileInterface + + +class ConanFileDependencies: + + def __init__(self): + self._nodes = [] # Graph algorithm guarantees not duplicated + self._get_ordered_breadth_first_cache = None # TODO: Replace functools Conan 2.0 + + @property + def nodes(self): + return self._nodes + + def add(self, conanfile): + # TODO: is it better to store the wrapped or the original one? + self._nodes.append(conanfile) + + def _get_ordered_breadth_first(self): + """ dummy example visitor that returns ordered breadth-first all the deps + """ + if self._get_ordered_breadth_first_cache is None: + result = OrderedDict() # TODO: this is a trick to get an ordered set + open_nodes = self._nodes + while open_nodes: + new_open = OrderedDict() + for n in open_nodes: + for d in n.dependencies.nodes: + new_open[d] = None + result[n] = None + open_nodes = [n for n in new_open if n not in result] + self._get_ordered_breadth_first_cache = result + return self._get_ordered_breadth_first_cache + + @property + def all(self): + return [ConanFileInterface(c) for c in self._get_ordered_breadth_first()] diff --git a/conans/model/conanfile_interface.py b/conans/model/conanfile_interface.py new file mode 100644 index 00000000000..b3394ed1f94 --- /dev/null +++ b/conans/model/conanfile_interface.py @@ -0,0 +1,22 @@ + + +class ConanFileInterface: + """ this is just a protective wrapper to give consumers + a limited view of conanfile dependencies, "read" only, + and only to some attributes, not methods + """ + + def __init__(self, conanfile): + self._conanfile = conanfile + + @property + def buildenv_info(self): + return self._conanfile.buildenv_info + + @property + def runenv_info(self): + return self._conanfile.runenv_info + + @property + def cpp_info(self): + return self._conanfile.cpp_info diff --git a/conans/model/profile.py b/conans/model/profile.py index 2d6ecd8c2dd..30b538b09c3 100644 --- a/conans/model/profile.py +++ b/conans/model/profile.py @@ -1,6 +1,7 @@ import copy from collections import OrderedDict, defaultdict +from conan.tools.env.environment import ProfileEnvironment from conans.client import settings_preprocessor from conans.errors import ConanException from conans.model.conf import ConfDefinition @@ -22,6 +23,7 @@ def __init__(self): self.options = OptionsValues() self.build_requires = OrderedDict() # ref pattern: list of ref self.conf = ConfDefinition() + self.buildenv = ProfileEnvironment() # Cached processed values self.processed_settings = None # Settings with values, and smart completion @@ -114,6 +116,7 @@ def compose(self, other): self.build_requires[pattern] = list(existing.values()) self.conf.update_conf_definition(other.conf) + self.buildenv.compose(other.buildenv) def update_settings(self, new_settings): """Mix the specified settings with the current profile. diff --git a/conans/test/functional/toolchains/test_env.py b/conans/test/functional/toolchains/test_env.py new file mode 100644 index 00000000000..276d855d21f --- /dev/null +++ b/conans/test/functional/toolchains/test_env.py @@ -0,0 +1,112 @@ +import os +import platform +import textwrap + +from conans.test.utils.test_files import temp_folder +from conans.test.utils.tools import TestClient +from conans.util.files import save + + +def test_complete(): + cmake = textwrap.dedent(r""" + import os + from conans import ConanFile + from conans.tools import save, chdir + class Pkg(ConanFile): + def package(self): + with chdir(self.package_folder): + save("mycmake.bat", "@echo off\necho MYCMAKE!!") + save("mycmake.sh", "@echo off\necho MYCMAKE!!") + os.chmod("mycmake.sh", 0o777) + + def package_info(self): + self.buildenv_info.prepend_path("PATH", self.package_folder) + """) + + gtest = textwrap.dedent(r""" + import os + from conans import ConanFile + from conans.tools import save, chdir + class Pkg(ConanFile): + def package(self): + with chdir(self.package_folder): + save("mygtest.bat", "@echo off\necho MYGTEST!!") + save("mygtest.sh", "@echo off\necho MYGTEST!!") + os.chmod("mygtest.sh", 0o777) + + def package_info(self): + self.buildenv_info.prepend_path("PATH", self.package_folder) + self.runenv_info.define("MYGTESTVAR", "MyGTestValue") + """) + client = TestClient() + client.save({"cmake/conanfile.py": cmake, + "gtest/conanfile.py": gtest}) + client.run("create cmake mycmake/0.1@") + client.run("create gtest mygtest/0.1@") + conanfile = textwrap.dedent(""" + from conans import ConanFile + class Pkg(ConanFile): + generators = "VirtualEnv" + build_requires = "mycmake/0.1" + + def build_requirements(self): + self.build_requires("mygtest/0.1", force_host_context=True) + """) + + # Some scripts in a random system folders, path adding to the profile [env] + tmp_folder = temp_folder() + save(os.path.join(tmp_folder, "mycompiler.bat"), "@echo off\n" + "echo MYCOMPILER!!\n" + "echo MYPATH=%PATH%") + save(os.path.join(tmp_folder, "mycompiler.sh"), "echo MYCOMPILER!!\n" + "echo MYPATH=$PATH") + os.chmod(os.path.join(tmp_folder, "mycompiler.sh"), 0o777) + tmp_folder2 = temp_folder() + save(os.path.join(tmp_folder2, "mycompiler.bat"), "@echo off\n" + "echo MYCOMPILER2!!\n" + "echo MYPATH2=%PATH%") + save(os.path.join(tmp_folder2, "mycompiler.sh"), "echo MYCOMPILER2!!\n" + "echo MYPATH2=$PATH") + os.chmod(os.path.join(tmp_folder2, "mycompiler.sh"), 0o777) + + myrunner_bat = "@echo off\necho MYGTESTVAR=%MYGTESTVAR%!!\n" + myrunner_sh = "echo MYGTESTVAR=$MYGTESTVAR!!\n" + + myprofile = textwrap.dedent(""" + [buildenv] + PATH+=(path){} + mypkg*:PATH=! + mypkg*:PATH+=(path){} + """.format(tmp_folder, tmp_folder2)) + client.save({"conanfile.py": conanfile, + "myprofile": myprofile, + "myrunner.bat": myrunner_bat, + "myrunner.sh": myrunner_sh}, clean_first=True) + os.chmod(os.path.join(client.current_folder, "myrunner.sh"), 0o777) + + client.run("install . -pr=myprofile") + # Run the BUILD environment + if platform.system() == "Windows": + client.run_command("buildenv.bat && mycmake.bat && mygtest.bat && mycompiler.bat") + else: + client.run_command('bash -c "source buildenv.sh && mycmake.sh && ' + 'mygtest.sh && mycompiler.sh"') + assert "MYCMAKE!!" in client.out + assert "MYCOMPILER!!" in client.out + assert "MYGTEST!!" in client.out + + # Run the RUN environment + if platform.system() == "Windows": + client.run_command("runenv.bat && myrunner.bat") + else: + client.run_command('bash -c "source runenv.sh && ./myrunner.sh"') + assert "MYGTESTVAR=MyGTestValue!!" in client.out + + # Now with pkg-specific env-var + client.run("install . mypkg/0.1@ -pr=myprofile") + if platform.system() == "Windows": + client.run_command('buildenv.bat && mycompiler.bat') + else: + client.run_command('bash -c "source buildenv.sh && mycompiler.sh"') + assert "MYCOMPILER2!!" in client.out + assert "MYPATH2=" in client.out diff --git a/conans/test/unittests/tools/env/__init__.py b/conans/test/unittests/tools/env/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/conans/test/unittests/tools/env/test_env.py b/conans/test/unittests/tools/env/test_env.py new file mode 100644 index 00000000000..18731bd28d8 --- /dev/null +++ b/conans/test/unittests/tools/env/test_env.py @@ -0,0 +1,212 @@ +import os +import platform +import subprocess +import textwrap + +import pytest + +from conan.tools.env import Environment +from conan.tools.env.environment import ProfileEnvironment +from conans.client.tools import chdir +from conans.test.utils.test_files import temp_folder +from conans.util.files import save + + +def test_compose(): + env = Environment() + env.define("MyVar", "MyValue") + env.define("MyVar2", "MyValue2") + env.define("MyVar3", "MyValue3") + env.define("MyVar4", "MyValue4") + env.unset("MyVar5") + + env2 = Environment() + env2.define("MyVar", "MyNewValue") + env2.append("MyVar2", "MyNewValue2") + env2.prepend("MyVar3", "MyNewValue3") + env2.unset("MyVar4") + env2.define("MyVar5", "MyNewValue5") + + env3 = env.compose(env2) + assert env3.value("MyVar") == "MyNewValue" + assert env3.value("MyVar2") == 'MyValue2 MyNewValue2' + assert env3.value("MyVar3") == 'MyNewValue3 MyValue3' + assert env3.value("MyVar4") == "" + assert env3.value("MyVar5") == 'MyNewValue5' + + +def test_profile(): + myprofile = textwrap.dedent(""" + # define + MyVar1=MyValue1 + # append + MyVar2+=MyValue2 + # multiple append works + MyVar2+=MyValue2_2 + # prepend + MyVar3=+MyValue3 + # unset + MyVar4=! + # Empty + MyVar5= + + # PATHS + # define + MyPath1=(path)/my/path1 + # append + MyPath2+=(path)/my/path2 + # multiple append works + MyPath2+=(path)/my/path2_2 + # prepend + MyPath3=+(path)/my/path3 + # unset + MyPath4=! + + # PER-PACKAGE + mypkg*:MyVar2=MyValue2 + """) + + profile_env = ProfileEnvironment() + profile_env.loads(myprofile) + env = profile_env.get_env("") + assert env.value("MyVar1") == "MyValue1" + assert env.value("MyVar2", "$MyVar2") == '$MyVar2 MyValue2 MyValue2_2' + assert env.value("MyVar3", "$MyVar3") == 'MyValue3 $MyVar3' + assert env.value("MyVar4") == "" + assert env.value("MyVar5") == '' + + env = profile_env.get_env("mypkg1/1.0") + assert env.value("MyVar1") == "MyValue1" + assert env.value("MyVar2", "$MyVar2") == 'MyValue2' + + +def test_env_files(): + env = Environment() + env.define("MyVar", "MyValue") + env.define("MyVar1", "MyValue1") + env.append("MyVar2", "MyValue2") + env.prepend("MyVar3", "MyValue3") + env.unset("MyVar4") + env.define("MyVar5", "MyValue5 With Space5=More Space5;:More") + env.append("MyVar6", "MyValue6") # Append, but previous not existing + env.define_path("MyPath1", "/Some/Path1/") + env.append_path("MyPath2", ["/Some/Path2/", "/Other/Path2/"]) + env.prepend_path("MyPath3", "/Some/Path3/") + env.unset("MyPath4") + folder = temp_folder() + + prevenv = {"MyVar1": "OldVar1", + "MyVar2": "OldVar2", + "MyVar3": "OldVar3", + "MyVar4": "OldVar4", + "MyPath1": "OldPath1", + "MyPath2": "OldPath2", + "MyPath3": "OldPath3", + "MyPath4": "OldPath4", + } + + display_bat = textwrap.dedent("""\ + @echo off + echo MyVar=%MyVar%!! + echo MyVar1=%MyVar1%!! + echo MyVar2=%MyVar2%!! + echo MyVar3=%MyVar3%!! + echo MyVar4=%MyVar4%!! + echo MyVar5=%MyVar5%!! + echo MyVar6=%MyVar6%!! + echo MyPath1=%MyPath1%!! + echo MyPath2=%MyPath2%!! + echo MyPath3=%MyPath3%!! + echo MyPath4=%MyPath4%!! + """) + + display_sh = textwrap.dedent("""\ + echo MyVar=$MyVar!! + echo MyVar1=$MyVar1!! + echo MyVar2=$MyVar2!! + echo MyVar3=$MyVar3!! + echo MyVar4=$MyVar4!! + echo MyVar5=$MyVar5!! + echo MyVar6=$MyVar6!! + echo MyPath1=$MyPath1!! + echo MyPath2=$MyPath2!! + echo MyPath3=$MyPath3!! + echo MyPath4=$MyPath4!! + """) + + def check(cmd_): + out, _ = subprocess.Popen(cmd_, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env=prevenv, shell=True).communicate() + out = out.decode() + assert "MyVar=MyValue!!" in out + assert "MyVar1=MyValue1!!" in out + assert "MyVar2=OldVar2 MyValue2!!" in out + assert "MyVar3=MyValue3 OldVar3!!" in out + assert "MyVar4=!!" in out + assert "MyVar5=MyValue5 With Space5=More Space5;:More!!" in out + assert "MyVar6= MyValue6!!" in out # The previous is non existing, append has space + assert "MyPath1=/Some/Path1/!!" in out + assert "MyPath2=OldPath2:/Some/Path2/:/Other/Path2/!!" in out + assert "MyPath3=/Some/Path3/:OldPath3!!" in out + assert "MyPath4=!!" in out + + # This should be output when deactivated + assert "MyVar=!!" in out + assert "MyVar1=OldVar1!!" in out + assert "MyVar2=OldVar2!!" in out + assert "MyVar3=OldVar3!!" in out + assert "MyVar4=OldVar4!!" in out + assert "MyVar5=!!" in out + assert "MyVar6=!!" in out + assert "MyPath1=OldPath1!!" in out + assert "MyPath2=OldPath2!!" in out + assert "MyPath3=OldPath3!!" in out + assert "MyPath4=OldPath4!!" in out + + with chdir(folder): + if platform.system() == "Windows": + env.save_bat("test.bat", pathsep=":") + save("display.bat", display_bat) + cmd = "test.bat && display.bat && deactivate_test.bat && display.bat" + check(cmd) + # FIXME: Powershell still not working + # env.save_ps1("test.ps1", pathsep=":") + # print(load("test.ps1")) + # cmd = r'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe' + # check(cmd) + else: + env.save_sh("test.sh") + save("display.sh", display_sh) + os.chmod("display.sh", 0o777) + cmd = 'bash -c ". test.sh && ./display.sh && . deactivate_test.sh && ./display.sh"' + check(cmd) + + +@pytest.mark.skipif(platform.system() != "Windows", reason="Requires Windows") +def test_windows_case_insensitive(): + # Append and define operation over the same variable in Windows preserve order + env = Environment() + env.define("MyVar", "MyValueA") + env.define("MYVAR", "MyValueB") + env.define("MyVar1", "MyValue1A") + env.append("MYVAR1", "MyValue1B") + folder = temp_folder() + + display_bat = textwrap.dedent("""\ + @echo off + echo MyVar=%MyVar%!! + echo MyVar1=%MyVar1%!! + """) + + with chdir(folder): + env.save_bat("test.bat") + save("display.bat", display_bat) + cmd = "test.bat && display.bat && deactivate_test.bat && display.bat" + out, _ = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + shell=True).communicate() + + out = out.decode() + assert "MyVar=MyValueB!!" in out + assert "MyVar=!!" in out + assert "MyVar1=MyValue1A MyValue1B!!" in out + assert "MyVar1=!!" in out From 3a585e593c4f6746e6d44bd452a00ae247a73d92 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 24 Feb 2021 00:04:23 +0100 Subject: [PATCH 02/28] build-host contexts poc --- conan/tools/env/virtualenv.py | 51 ++++-- conans/client/graph/graph_builder.py | 2 + conans/model/conanfile_interface.py | 8 + conans/test/functional/toolchains/test_env.py | 155 +++++++++++------- 4 files changed, 142 insertions(+), 74 deletions(-) diff --git a/conan/tools/env/virtualenv.py b/conan/tools/env/virtualenv.py index dd016aae28d..79135dd120c 100644 --- a/conan/tools/env/virtualenv.py +++ b/conan/tools/env/virtualenv.py @@ -1,6 +1,7 @@ import platform from conan.tools.env import Environment +from conans.client.graph.graph import CONTEXT_BUILD class VirtualEnv: @@ -18,16 +19,21 @@ def build_environment(self): # Visit all dependencies deps_env = Environment() # TODO: The visitor of dependencies needs to be implemented correctly + # TODO: The environment should probably be composed with direct dependencies first + # TODO: in paths, but this is the opposite for dep in self._conanfile.dependencies.all: # environ_info is always "build" dep_env = dep.buildenv_info if dep_env is not None: - deps_env = deps_env.compose(dep_env) + deps_env.compose(dep_env) + if dep.context == CONTEXT_BUILD: + runenv = self._runenv_from_cpp_info(dep) + deps_env.compose(runenv) # The profile environment has precedence, applied last profile_env = self._conanfile.buildenv - result = deps_env.compose(profile_env) - return result + deps_env.compose(profile_env) + return deps_env def autorun_environment(self): """ automatically collects the runtime environment from 'cpp_info' from dependencies @@ -35,19 +41,30 @@ def autorun_environment(self): by parameter or [conf] """ dyn_runenv = Environment() - # TODO: New cpp_info access for dep in self._conanfile.dependencies.all: - cpp_info = dep.cpp_info - if cpp_info.exes: - dyn_runenv.append_path("PATH", cpp_info.bin_paths) - os_ = self._conanfile.settings.get_safe("os") - if cpp_info.lib_paths: - if os_ == "Linux": - dyn_runenv.append_path("LD_LIBRARY_PATH", cpp_info.lib_paths) - elif os_ == "Macos": - dyn_runenv.append_path("DYLD_LIBRARY_PATH", cpp_info.lib_paths) - if cpp_info.framework_paths: - dyn_runenv.append_path("DYLD_FRAMEWORK_PATH", cpp_info.framework_paths) + if dep.context == CONTEXT_BUILD: # Build environment cannot happen in runtime + continue + env = self._runenv_from_cpp_info(dep) + dyn_runenv.compose(env) + return dyn_runenv + + @staticmethod + def _runenv_from_cpp_info(conanfile_dep): + """ return an Environment deducing the runtime information from a cpp_info + """ + dyn_runenv = Environment() + cpp_info = conanfile_dep.cpp_info + if cpp_info.exes: + dyn_runenv.prepend_path("PATH", cpp_info.bin_paths) + # If it is a build_require this will be the build-os, otherwise it will be the host-os + os_ = conanfile_dep.settings.get_safe("os") + if cpp_info.lib_paths: + if os_ == "Linux": + dyn_runenv.prepend_path("LD_LIBRARY_PATH", cpp_info.lib_paths) + elif os_ == "Macos": + dyn_runenv.prepend_path("DYLD_LIBRARY_PATH", cpp_info.lib_paths) + if cpp_info.framework_paths: + dyn_runenv.prepend_path("DYLD_FRAMEWORK_PATH", cpp_info.framework_paths) return dyn_runenv def run_environment(self): @@ -60,10 +77,10 @@ def run_environment(self): # run_environ_info is always "host" dep_env = dep.runenv_info if dep_env is not None: - deps_env = deps_env.compose(dep_env) + deps_env.compose(dep_env) autorun = self.autorun_environment() - deps_env = deps_env.compose(autorun) + deps_env.compose(autorun) # FIXME: Missing profile info result = deps_env return result diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py index 45b866a60b3..b33a74e60f1 100644 --- a/conans/client/graph/graph_builder.py +++ b/conans/client/graph/graph_builder.py @@ -53,6 +53,7 @@ def load_graph(self, root_node, check_updates, update, remotes, profile_host, pr initial = graph_lock.initial_counter if graph_lock else None dep_graph = DepsGraph(initial_node_id=initial) # compute the conanfile entry point for this dependency graph + root_node.conanfile.context = CONTEXT_HOST # FIXME: Is this always true? root_node.public_closure.add(root_node) root_node.public_deps.add(root_node) root_node.transitive_closure[root_node.name] = root_node @@ -205,6 +206,7 @@ def _expand_require(self, require, node, graph, check_updates, update, remotes, remotes, profile_host, profile_build, graph_lock, context_switch=context_switch, populate_settings_target=populate_settings_target) + new_node.conanfile._conan_context = context node.conanfile.dependencies.add(new_node.conanfile) # The closure of a new node starts with just itself new_node.public_closure.add(new_node) diff --git a/conans/model/conanfile_interface.py b/conans/model/conanfile_interface.py index b3394ed1f94..194f634c4e4 100644 --- a/conans/model/conanfile_interface.py +++ b/conans/model/conanfile_interface.py @@ -20,3 +20,11 @@ def runenv_info(self): @property def cpp_info(self): return self._conanfile.cpp_info + + @property + def settings(self): + return self._conanfile.settings + + @property + def context(self): + return self._conanfile._conan_context diff --git a/conans/test/functional/toolchains/test_env.py b/conans/test/functional/toolchains/test_env.py index 276d855d21f..19072e3d6e8 100644 --- a/conans/test/functional/toolchains/test_env.py +++ b/conans/test/functional/toolchains/test_env.py @@ -2,25 +2,46 @@ import platform import textwrap -from conans.test.utils.test_files import temp_folder +import pytest + +from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.tools import TestClient -from conans.util.files import save -def test_complete(): +@pytest.fixture() +def client(): + openssl = textwrap.dedent(r""" + import os + from conans import ConanFile + from conans.tools import save, chdir + class Pkg(ConanFile): + settings = "os" + def package(self): + with chdir(self.package_folder): + echo = "@echo off\necho MYOPENSSL={}!!".format(self.settings.os) + save("bin/myopenssl.bat", echo) + save("bin/myopenssl.sh", echo) + os.chmod("bin/myopenssl.sh", 0o777) + """) + cmake = textwrap.dedent(r""" import os from conans import ConanFile from conans.tools import save, chdir class Pkg(ConanFile): + settings = "os" + requires = "openssl/1.0" def package(self): with chdir(self.package_folder): - save("mycmake.bat", "@echo off\necho MYCMAKE!!") - save("mycmake.sh", "@echo off\necho MYCMAKE!!") + echo = "@echo off\necho MYCMAKE={}!!".format(self.settings.os) + save("mycmake.bat", echo + "\ncall myopenssl.bat") + save("mycmake.sh", echo + "\n myopenssl.sh") os.chmod("mycmake.sh", 0o777) def package_info(self): + # Custom buildenv not defined by cpp_info self.buildenv_info.prepend_path("PATH", self.package_folder) + self.buildenv_info.define("MYCMAKEVAR", "MYCMAKEVALUE!!") """) gtest = textwrap.dedent(r""" @@ -28,82 +49,102 @@ def package_info(self): from conans import ConanFile from conans.tools import save, chdir class Pkg(ConanFile): + settings = "os" def package(self): with chdir(self.package_folder): - save("mygtest.bat", "@echo off\necho MYGTEST!!") - save("mygtest.sh", "@echo off\necho MYGTEST!!") - os.chmod("mygtest.sh", 0o777) + echo = "@echo off\necho MYGTEST={}!!".format(self.settings.os) + save("bin/mygtest.bat", echo) + save("bin/mygtest.sh", echo) + os.chmod("bin/mygtest.sh", 0o777) def package_info(self): - self.buildenv_info.prepend_path("PATH", self.package_folder) - self.runenv_info.define("MYGTESTVAR", "MyGTestValue") + self.runenv_info.define("MYGTESTVAR", "MyGTestValue{}".format(self.settings.os)) """) client = TestClient() client.save({"cmake/conanfile.py": cmake, - "gtest/conanfile.py": gtest}) - client.run("create cmake mycmake/0.1@") - client.run("create gtest mygtest/0.1@") + "gtest/conanfile.py": gtest, + "openssl/conanfile.py": openssl}) + + client.run("export openssl openssl/1.0@") + client.run("export cmake mycmake/1.0@") + client.run("export gtest mygtest/1.0@") + + myrunner_bat = "@echo off\necho MYGTESTVAR=%MYGTESTVAR%!!\n" + myrunner_sh = "echo MYGTESTVAR=$MYGTESTVAR!!\n" + client.save({"myrunner.bat": myrunner_bat, + "myrunner.sh": myrunner_sh}, clean_first=True) + os.chmod(os.path.join(client.current_folder, "myrunner.sh"), 0o777) + return client + + +def test_complete(client): conanfile = textwrap.dedent(""" - from conans import ConanFile - class Pkg(ConanFile): - generators = "VirtualEnv" - build_requires = "mycmake/0.1" + from conans import ConanFile + class Pkg(ConanFile): + generators = "VirtualEnv" + requires = "openssl/1.0" + build_requires = "mycmake/1.0" - def build_requirements(self): - self.build_requires("mygtest/0.1", force_host_context=True) - """) + def build_requirements(self): + self.build_requires("mygtest/1.0", force_host_context=True) + """) + client.save({"conanfile.py": conanfile}) + client.run("install . -s:b os=Windows -s:h os=Linux --build=missing") + # Run the BUILD environment + if platform.system() == "Windows": + client.run_command("buildenv.bat && mycmake.bat") + else: + client.run_command('bash -c "source buildenv.sh && mycmake.sh"') + assert "MYCMAKE=Windows!!" in client.out + assert "MYOPENSSL=Windows!!" in client.out + + # Run the RUN environment + if platform.system() == "Windows": + client.run_command("runenv.bat && mygtest.bat && myrunner.bat") + else: + client.run_command('bash -c "source runenv.sh && mygtest.sh && ./myrunner.sh"') + assert "MYGTEST=Linux!!" in client.out + assert "MYGTESTVAR=MyGTestValueLinux!!" in client.out + + +def test_profile_buildenv(client): + conanfile = GenConanfile().with_generator("VirtualEnv") # Some scripts in a random system folders, path adding to the profile [env] - tmp_folder = temp_folder() - save(os.path.join(tmp_folder, "mycompiler.bat"), "@echo off\n" - "echo MYCOMPILER!!\n" - "echo MYPATH=%PATH%") - save(os.path.join(tmp_folder, "mycompiler.sh"), "echo MYCOMPILER!!\n" - "echo MYPATH=$PATH") - os.chmod(os.path.join(tmp_folder, "mycompiler.sh"), 0o777) - tmp_folder2 = temp_folder() - save(os.path.join(tmp_folder2, "mycompiler.bat"), "@echo off\n" - "echo MYCOMPILER2!!\n" - "echo MYPATH2=%PATH%") - save(os.path.join(tmp_folder2, "mycompiler.sh"), "echo MYCOMPILER2!!\n" - "echo MYPATH2=$PATH") - os.chmod(os.path.join(tmp_folder2, "mycompiler.sh"), 0o777) - myrunner_bat = "@echo off\necho MYGTESTVAR=%MYGTESTVAR%!!\n" - myrunner_sh = "echo MYGTESTVAR=$MYGTESTVAR!!\n" + compiler_bat = "@echo off\necho MYCOMPILER!!\necho MYPATH=%PATH%" + compiler_sh = "echo MYCOMPILER!!\necho MYPATH=$PATH" + compiler2_bat = "@echo off\necho MYCOMPILER2!!\necho MYPATH2=%PATH%" + compiler2_sh = "echo MYCOMPILER2!!\necho MYPATH2=$PATH" myprofile = textwrap.dedent(""" - [buildenv] - PATH+=(path){} - mypkg*:PATH=! - mypkg*:PATH+=(path){} - """.format(tmp_folder, tmp_folder2)) + [buildenv] + PATH+=(path){} + mypkg*:PATH=! + mypkg*:PATH+=(path){} + """.format(os.path.join(client.current_folder, "compiler"), + os.path.join(client.current_folder, "compiler2"))) client.save({"conanfile.py": conanfile, "myprofile": myprofile, - "myrunner.bat": myrunner_bat, - "myrunner.sh": myrunner_sh}, clean_first=True) - os.chmod(os.path.join(client.current_folder, "myrunner.sh"), 0o777) + "compiler/mycompiler.bat": compiler_bat, + "compiler/mycompiler.sh": compiler_sh, + "compiler2/mycompiler.bat": compiler2_bat, + "compiler2/mycompiler.sh": compiler2_sh}) + + os.chmod(os.path.join(client.current_folder, "compiler", "mycompiler.sh"), 0o777) + os.chmod(os.path.join(client.current_folder, "compiler2", "mycompiler.sh"), 0o777) client.run("install . -pr=myprofile") # Run the BUILD environment if platform.system() == "Windows": - client.run_command("buildenv.bat && mycmake.bat && mygtest.bat && mycompiler.bat") + client.run_command("buildenv.bat && mycompiler.bat") else: - client.run_command('bash -c "source buildenv.sh && mycmake.sh && ' - 'mygtest.sh && mycompiler.sh"') - assert "MYCMAKE!!" in client.out + client.run_command('bash -c "source buildenv.sh && mycompiler.sh"') assert "MYCOMPILER!!" in client.out - assert "MYGTEST!!" in client.out - - # Run the RUN environment - if platform.system() == "Windows": - client.run_command("runenv.bat && myrunner.bat") - else: - client.run_command('bash -c "source runenv.sh && ./myrunner.sh"') - assert "MYGTESTVAR=MyGTestValue!!" in client.out + assert "MYPATH=" in client.out # Now with pkg-specific env-var - client.run("install . mypkg/0.1@ -pr=myprofile") + client.run("install . mypkg/1.0@ -pr=myprofile") if platform.system() == "Windows": client.run_command('buildenv.bat && mycompiler.bat') else: From 5f323b1ac58ced09997629f5d882e03da95177d8 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 24 Feb 2021 11:32:40 +0100 Subject: [PATCH 03/28] added CMakeGen integration --- conan/tools/cmake/__init__.py | 1 + conan/tools/cmake/cmake.py | 12 ++++ conan/tools/cmake/cmakegen.py | 14 +++++ conan/tools/env/environment.py | 8 +++ conans/client/generators/__init__.py | 8 ++- .../toolchains/cmake/test_cmakegen.py | 62 +++++++++++++++++++ conans/test/functional/toolchains/test_env.py | 28 ++++----- 7 files changed, 115 insertions(+), 18 deletions(-) create mode 100644 conan/tools/cmake/cmakegen.py create mode 100644 conans/test/functional/toolchains/cmake/test_cmakegen.py diff --git a/conan/tools/cmake/__init__.py b/conan/tools/cmake/__init__.py index da8a68cc619..f500c9576d7 100644 --- a/conan/tools/cmake/__init__.py +++ b/conan/tools/cmake/__init__.py @@ -1,3 +1,4 @@ from conan.tools.cmake.toolchain import CMakeToolchain from conan.tools.cmake.cmake import CMake from conan.tools.cmake.cmakedeps import CMakeDeps +from conan.tools.cmake.cmakegen import CMakeGen diff --git a/conan/tools/cmake/cmake.py b/conan/tools/cmake/cmake.py index 4139447819d..c45c21b24f5 100644 --- a/conan/tools/cmake/cmake.py +++ b/conan/tools/cmake/cmake.py @@ -3,6 +3,7 @@ from conan.tools.cmake.base import CMakeToolchainBase from conan.tools.cmake.utils import get_generator, is_multi_configuration +from conan.tools.env.environment import environment_wrap_command from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg from conans.client import tools from conans.client.build import join_arguments @@ -82,6 +83,11 @@ def configure(self, source_folder=None): generator = '-G "{}" '.format(self._generator) if self._generator else "" command = "%s %s%s" % (self._cmake_program, generator, arg_list) + # Need to activate the buildenv if existing + env_filename = "buildenv.bat" if platform.system() == "Windows" else "buildenv.sh" + if os.path.isfile(env_filename): + command = environment_wrap_command(env_filename, command) + is_windows_mingw = platform.system() == "Windows" and self._generator == "MinGW Makefiles" self._conanfile.output.info("CMake command: %s" % command) with chdir(build_folder): @@ -118,6 +124,12 @@ def _build(self, build_type=None, target=None): arg_list = [args_to_string([bf]), build_config, args_to_string(args)] command = "%s --build %s" % (self._cmake_program, join_arguments(arg_list)) + + # Need to activate the buildenv if existing + env_filename = "buildenv.bat" if platform.system() == "Windows" else "buildenv.sh" + if os.path.isfile(env_filename): + command = environment_wrap_command(env_filename, command) + self._conanfile.output.info("CMake command: %s" % command) self._conanfile.run(command) diff --git a/conan/tools/cmake/cmakegen.py b/conan/tools/cmake/cmakegen.py new file mode 100644 index 00000000000..98859b9f8cc --- /dev/null +++ b/conan/tools/cmake/cmakegen.py @@ -0,0 +1,14 @@ +from conan.tools.cmake import CMakeDeps, CMakeToolchain +from conan.tools.env import VirtualEnv + + +class CMakeGen: + def __init__(self, conanfile): + self.toolchain = CMakeToolchain(conanfile) + self.deps = CMakeDeps(conanfile) + self.env = VirtualEnv(conanfile) + + def generate(self): + self.toolchain.generate() + self.deps.generate() + self.env.generate() diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index 83b4146356a..838a4f5bbe9 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -10,6 +10,14 @@ _PATHSEP = "$CONAN_PATHSEP%" +def environment_wrap_command(filename, cmd): + if filename.endswith(".bat"): + return "{} && {}".format(filename, cmd) + elif filename.endswith(".sh"): + return 'bash -c ". {} && {}"'.format(filename, cmd.replace('"', r'\"')) + raise Exception("Unsupported environment file type {}".format(filename)) + + class Environment: def __init__(self): # TODO: Maybe we need to pass conanfile to get the [conf] diff --git a/conans/client/generators/__init__.py b/conans/client/generators/__init__.py index 6f4f3d06a4f..5c5326d8c80 100644 --- a/conans/client/generators/__init__.py +++ b/conans/client/generators/__init__.py @@ -65,7 +65,8 @@ def __init__(self): "make": MakeGenerator, "deploy": DeployGenerator, "markdown": MarkdownGenerator} - self._new_generators = ["CMakeToolchain", "CMakeDeps", "MakeToolchain", "MSBuildToolchain", + self._new_generators = ["CMakeGen", "CMakeToolchain", "CMakeDeps", + "MakeToolchain", "MSBuildToolchain", "MesonToolchain", "MSBuildDeps", "QbsToolchain", "msbuild", "VirtualEnv"] @@ -90,7 +91,10 @@ def _new_generator(self, generator_name, output): if generator_name == "CMakeToolchain": from conan.tools.cmake import CMakeToolchain return CMakeToolchain - if generator_name == "CMakeDeps": + elif generator_name == "CMakeGen": + from conan.tools.cmake import CMakeGen + return CMakeGen + elif generator_name == "CMakeDeps": from conan.tools.cmake import CMakeDeps return CMakeDeps elif generator_name == "MakeToolchain": diff --git a/conans/test/functional/toolchains/cmake/test_cmakegen.py b/conans/test/functional/toolchains/cmake/test_cmakegen.py new file mode 100644 index 00000000000..fda1d03604b --- /dev/null +++ b/conans/test/functional/toolchains/cmake/test_cmakegen.py @@ -0,0 +1,62 @@ +import textwrap + +from conans.test.assets.sources import gen_function_cpp +from conans.test.utils.tools import TestClient + + +def test_cmakegen(): + client = TestClient() + client.run("new hello/0.1 -s") + client.run("create .") + + cmakewrapper = textwrap.dedent(r""" + from conans import ConanFile + import os + from conans.tools import save, chdir + class Pkg(ConanFile): + def package(self): + with chdir(self.package_folder): + save("cmake.bat", "@echo off\necho MYCMAKE WRAPPER!!\ncmake.exe %*") + save("cmake", 'echo MYCMAKE WRAPPER!!\n/usr/bin/cmake "$@"') + os.chmod("cmake", 0o777) + + def package_info(self): + # Custom buildenv not defined by cpp_info + self.buildenv_info.prepend_path("PATH", self.package_folder) + """) + consumer = textwrap.dedent(""" + from conans import ConanFile + from conan.tools.cmake import CMake + class App(ConanFile): + settings = "os", "arch", "compiler", "build_type" + exports_sources = "CMakeLists.txt", "main.cpp" + requires = "hello/0.1" + build_requires = "cmakewrapper/0.1" + generators = "CMakeGen" + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + """) + + cmakelists = textwrap.dedent(""" + cmake_minimum_required(VERSION 3.15) + project(MyApp CXX) + + find_package(hello) + add_executable(app main.cpp) + target_link_libraries(app hello::hello) + """) + + client.save({"cmakewrapper/conanfile.py": cmakewrapper, + "consumer/conanfile.py": consumer, + "consumer/main.cpp": gen_function_cpp(name="main", includes=["hello"], + calls=["hello"]), + "consumer/CMakeLists.txt": cmakelists}, + clean_first=True) + + client.run("create cmakewrapper cmakewrapper/0.1@") + client.run("create consumer consumer/0.1@") + assert "MYCMAKE WRAPPER!!" in client.out + assert "consumer/0.1: Created package" in client.out diff --git a/conans/test/functional/toolchains/test_env.py b/conans/test/functional/toolchains/test_env.py index 19072e3d6e8..70195a166db 100644 --- a/conans/test/functional/toolchains/test_env.py +++ b/conans/test/functional/toolchains/test_env.py @@ -4,6 +4,7 @@ import pytest +from conan.tools.env.environment import environment_wrap_command from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.tools import TestClient @@ -92,18 +93,17 @@ def build_requirements(self): client.save({"conanfile.py": conanfile}) client.run("install . -s:b os=Windows -s:h os=Linux --build=missing") # Run the BUILD environment - if platform.system() == "Windows": - client.run_command("buildenv.bat && mycmake.bat") - else: - client.run_command('bash -c "source buildenv.sh && mycmake.sh"') + ext = "bat" if platform.system() == "Windows" else "sh" # TODO: Decide on logic .bat vs .sh + cmd = environment_wrap_command("buildenv.{}".format(ext), "mycmake.{}".format(ext)) + client.run_command(cmd) assert "MYCMAKE=Windows!!" in client.out assert "MYOPENSSL=Windows!!" in client.out # Run the RUN environment - if platform.system() == "Windows": - client.run_command("runenv.bat && mygtest.bat && myrunner.bat") - else: - client.run_command('bash -c "source runenv.sh && mygtest.sh && ./myrunner.sh"') + cmd = environment_wrap_command("runenv.{}".format(ext), + "mygtest.{ext} && .{sep}myrunner.{ext}".format(ext=ext, + sep=os.sep)) + client.run_command(cmd) assert "MYGTEST=Linux!!" in client.out assert "MYGTESTVAR=MyGTestValueLinux!!" in client.out @@ -136,18 +136,14 @@ def test_profile_buildenv(client): client.run("install . -pr=myprofile") # Run the BUILD environment - if platform.system() == "Windows": - client.run_command("buildenv.bat && mycompiler.bat") - else: - client.run_command('bash -c "source buildenv.sh && mycompiler.sh"') + ext = "bat" if platform.system() == "Windows" else "sh" # TODO: Decide on logic .bat vs .sh + cmd = environment_wrap_command("buildenv.{}".format(ext), "mycompiler.{}".format(ext)) + client.run_command(cmd) assert "MYCOMPILER!!" in client.out assert "MYPATH=" in client.out # Now with pkg-specific env-var client.run("install . mypkg/1.0@ -pr=myprofile") - if platform.system() == "Windows": - client.run_command('buildenv.bat && mycompiler.bat') - else: - client.run_command('bash -c "source buildenv.sh && mycompiler.sh"') + client.run_command(cmd) assert "MYCOMPILER2!!" in client.out assert "MYPATH2=" in client.out From 16863a17f9c5c679762a11c3736832e8c894556c Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 24 Feb 2021 14:17:42 +0100 Subject: [PATCH 04/28] fixing Macos tests --- conan/tools/cmake/cmake.py | 1 + conans/test/functional/toolchains/cmake/test_cmakegen.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/conan/tools/cmake/cmake.py b/conan/tools/cmake/cmake.py index c45c21b24f5..9dfc3cab244 100644 --- a/conan/tools/cmake/cmake.py +++ b/conan/tools/cmake/cmake.py @@ -92,6 +92,7 @@ def configure(self, source_folder=None): self._conanfile.output.info("CMake command: %s" % command) with chdir(build_folder): if is_windows_mingw: + # FIXME: Remove this and use -DCMAKE_SH="CMAKE_SH-NOTFOUND" (in toolchain) with tools.remove_from_path("sh"): self._conanfile.run(command) else: diff --git a/conans/test/functional/toolchains/cmake/test_cmakegen.py b/conans/test/functional/toolchains/cmake/test_cmakegen.py index fda1d03604b..df4be960e2a 100644 --- a/conans/test/functional/toolchains/cmake/test_cmakegen.py +++ b/conans/test/functional/toolchains/cmake/test_cmakegen.py @@ -17,8 +17,8 @@ class Pkg(ConanFile): def package(self): with chdir(self.package_folder): save("cmake.bat", "@echo off\necho MYCMAKE WRAPPER!!\ncmake.exe %*") - save("cmake", 'echo MYCMAKE WRAPPER!!\n/usr/bin/cmake "$@"') - os.chmod("cmake", 0o777) + save("cmake.sh", 'echo MYCMAKE WRAPPER!!\ncmake "$@"') + os.chmod("cmake.sh", 0o777) def package_info(self): # Custom buildenv not defined by cpp_info @@ -36,6 +36,8 @@ class App(ConanFile): def build(self): cmake = CMake(self) + if self.settings.os != "Windows": + cmake._cmake_program = "cmake.sh" # VERY DIRTY HACK cmake.configure() cmake.build() """) From 3fd4633936d60d6a0e0a10f34a669ae26592cbf4 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 24 Feb 2021 16:46:01 +0100 Subject: [PATCH 05/28] review --- conan/tools/env/environment.py | 49 ++++++++---- conans/test/unittests/tools/env/test_env.py | 83 +++++++++++++++++++-- 2 files changed, 111 insertions(+), 21 deletions(-) diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index 838a4f5bbe9..1879bf0bcda 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -6,8 +6,17 @@ from conans.errors import ConanException from conans.util.files import save -_ENV_VAR_PLACEHOLDER = "$PREVIOUS_ENV_VAR_VALUE%" -_PATHSEP = "$CONAN_PATHSEP%" + +class _EnvVarPlaceHolder: + pass + + +class _Sep(str): + pass + + +class _PathSep: + pass def environment_wrap_command(filename, cmd): @@ -27,16 +36,17 @@ def __init__(self): def vars(self): return list(self._values.keys()) - def value(self, name, placeholder="{}", pathsep=os.pathsep): + def value(self, name, placeholder="{name}", pathsep=os.pathsep): return self._format_value(name, self._values[name], placeholder, pathsep) @staticmethod def _format_value(name, varvalues, placeholder, pathsep): values = [] for v in varvalues: - if v == _ENV_VAR_PLACEHOLDER: + + if v is _EnvVarPlaceHolder: values.append(placeholder.format(name=name)) - elif v == _PATHSEP: + elif v is _PathSep: values.append(pathsep) else: values.append(v) @@ -55,11 +65,12 @@ def _list_value(value, separator): return [value] def define(self, name, value, separator=" "): - value = self._list_value(value, separator) + value = self._list_value(value, _Sep(separator)) self._values[name] = value def define_path(self, name, value): - self.define(name, value, _PATHSEP) + value = self._list_value(value, _PathSep) + self._values[name] = value def unset(self, name): """ @@ -68,18 +79,20 @@ def unset(self, name): self._values[name] = [] def append(self, name, value, separator=" "): - value = self._list_value(value, separator) - self._values[name] = [_ENV_VAR_PLACEHOLDER] + [separator] + value + value = self._list_value(value, _Sep(separator)) + self._values[name] = [_EnvVarPlaceHolder] + [_Sep(separator)] + value def append_path(self, name, value): - self.append(name, value, _PATHSEP) + value = self._list_value(value, _PathSep) + self._values[name] = [_EnvVarPlaceHolder] + [_PathSep] + value def prepend(self, name, value, separator=" "): - value = self._list_value(value, separator) - self._values[name] = value + [separator] + [_ENV_VAR_PLACEHOLDER] + value = self._list_value(value, _Sep(separator)) + self._values[name] = value + [_Sep(separator)] + [_EnvVarPlaceHolder] def prepend_path(self, name, value): - self.prepend(name, value, _PATHSEP) + value = self._list_value(value, _PathSep) + self._values[name] = value + [_PathSep] + [_EnvVarPlaceHolder] def save_bat(self, filename, generate_deactivate=True, pathsep=os.pathsep): deactivate = textwrap.dedent("""\ @@ -167,14 +180,20 @@ def compose(self, other): self._values[k] = v else: try: - index = v.index(_ENV_VAR_PLACEHOLDER) + index = v.index(_EnvVarPlaceHolder) except ValueError: # The other doesn't have placeholder, overwrites self._values[k] = v else: new_value = v[:] # do a copy new_value[index:index + 1] = existing # replace the placeholder + # Trim front and back separators + val = new_value[0] + if isinstance(val, _Sep) or val is _PathSep: + new_value = new_value[1:] + val = new_value[-1] + if isinstance(val, _Sep) or val is _PathSep: + new_value = new_value[:-1] self._values[k] = new_value - return self diff --git a/conans/test/unittests/tools/env/test_env.py b/conans/test/unittests/tools/env/test_env.py index 18731bd28d8..a7ded847d92 100644 --- a/conans/test/unittests/tools/env/test_env.py +++ b/conans/test/unittests/tools/env/test_env.py @@ -27,12 +27,83 @@ def test_compose(): env2.unset("MyVar4") env2.define("MyVar5", "MyNewValue5") - env3 = env.compose(env2) - assert env3.value("MyVar") == "MyNewValue" - assert env3.value("MyVar2") == 'MyValue2 MyNewValue2' - assert env3.value("MyVar3") == 'MyNewValue3 MyValue3' - assert env3.value("MyVar4") == "" - assert env3.value("MyVar5") == 'MyNewValue5' + env.compose(env2) + assert env.value("MyVar") == "MyNewValue" + assert env.value("MyVar2") == 'MyValue2 MyNewValue2' + assert env.value("MyVar3") == 'MyNewValue3 MyValue3' + assert env.value("MyVar4") == "" + assert env.value("MyVar5") == 'MyNewValue5' + + +@pytest.mark.parametrize("op1, v1, s1, op2, v2, s2, result", + [("define", "Val1", " ", "define", "Val2", " ", "Val2"), + ("define", "Val1", " ", "append", "Val2", " ", "Val1 Val2"), + ("define", "Val1", " ", "prepend", "Val2", " ", "Val2 Val1"), + ("define", "Val1", " ", "unset", "", " ", ""), + ("append", "Val1", " ", "define", "Val2", " ", "Val2"), + ("append", "Val1", " ", "append", "Val2", " ", "MyVar Val1 Val2"), + ("append", "Val1", " ", "prepend", "Val2", " ", "Val2 MyVar Val1"), + ("append", "Val1", " ", "unset", "", " ", ""), + ("prepend", "Val1", " ", "define", "Val2", " ", "Val2"), + ("prepend", "Val1", " ", "append", "Val2", " ", "Val1 MyVar Val2"), + ("prepend", "Val1", " ", "prepend", "Val2", " ", "Val2 Val1 MyVar"), + ("prepend", "Val1", " ", "unset", "", " ", ""), + ("unset", "", " ", "define", "Val2", " ", "Val2"), + ("unset", "", " ", "append", "Val2", " ", "Val2"), + ("unset", "", " ", "prepend", "Val2", " ", "Val2"), + ("unset", "", " ", "unset", "", " ", ""), + # different separators + ("append", "Val1", "+", "append", "Val2", "-", "MyVar+Val1-Val2"), + ("append", "Val1", "+", "prepend", "Val2", "-", "Val2-MyVar+Val1"), + ("unset", "", " ", "append", "Val2", "+", "Val2"), + ("unset", "", " ", "prepend", "Val2", "+", "Val2"), + ]) +def test_compose_combinations(op1, v1, s1, op2, v2, s2, result): + env = Environment() + if op1 != "unset": + getattr(env, op1)("MyVar", v1, s1) + else: + env.unset("MyVar") + env2 = Environment() + if op2 != "unset": + getattr(env2, op2)("MyVar", v2, s2) + else: + env2.unset("MyVar") + env.compose(env2) + assert env.value("MyVar") == result + + +@pytest.mark.parametrize("op1, v1, op2, v2, result", + [("define", "/path1", "define", "/path2", "/path2"), + ("define", "/path1", "append", "/path2", "/path1:/path2"), + ("define", "/path1", "prepend", "/path2", "/path2:/path1"), + ("define", "/path1", "unset", "", ""), + ("append", "/path1", "define", "/path2", "/path2"), + ("append", "/path1", "append", "/path2", "MyVar:/path1:/path2"), + ("append", "/path1", "prepend", "/path2", "/path2:MyVar:/path1"), + ("append", "/path1", "unset", "", ""), + ("prepend", "/path1", "define", "/path2", "/path2"), + ("prepend", "/path1", "append", "/path2", "/path1:MyVar:/path2"), + ("prepend", "/path1", "prepend", "/path2", "/path2:/path1:MyVar"), + ("prepend", "/path1", "unset", "", ""), + ("unset", "", "define", "/path2", "/path2"), + ("unset", "", "append", "/path2", "/path2"), + ("unset", "", "prepend", "/path2", "/path2"), + ("unset", "", "unset", "", ""), + ]) +def test_compose_path_combinations(op1, v1, op2, v2, result): + env = Environment() + if op1 != "unset": + getattr(env, op1+"_path")("MyVar", v1) + else: + env.unset("MyVar") + env2 = Environment() + if op2 != "unset": + getattr(env2, op2+"_path")("MyVar", v2) + else: + env2.unset("MyVar") + env.compose(env2) + assert env.value("MyVar", pathsep=":") == result def test_profile(): From de6787ef6b09625edf99b0304abae20fc5448b5c Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 24 Feb 2021 20:10:32 +0100 Subject: [PATCH 06/28] make sure profile compose and include() correctly --- conan/tools/env/environment.py | 10 +++- conans/client/profile_loader.py | 2 +- .../environment/test_buildenv_profile.py | 53 +++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 conans/test/integration/environment/test_buildenv_profile.py diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index 1879bf0bcda..b4743c78074 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -33,6 +33,9 @@ def __init__(self): # It being ordered allows for Windows case-insensitive composition self._values = OrderedDict() # {var_name: [] of values, including separators} + def __repr__(self): + return repr(self._values) + def vars(self): return list(self._values.keys()) @@ -201,8 +204,13 @@ class ProfileEnvironment: def __init__(self): self._environments = OrderedDict() + def __repr__(self): + return repr(self._environments) + def get_env(self, ref): - # TODO: Maybe we want to make this lazy, so this is not evaluated for every package + """ computes package-specific Environment + it is only called when conanfile.buildenv is called + """ result = Environment() for pattern, env in self._environments.items(): if pattern is None or fnmatch.fnmatch(str(ref), pattern): diff --git a/conans/client/profile_loader.py b/conans/client/profile_loader.py index a0f83fbca19..e3c0b7f50cd 100644 --- a/conans/client/profile_loader.py +++ b/conans/client/profile_loader.py @@ -230,7 +230,7 @@ def get_package_name_value(item): buildenv = ProfileEnvironment() buildenv.loads(doc.buildenv) # TODO: compose ProfileEnvironment - base_profile.buildenv = buildenv + base_profile.buildenv.compose(buildenv) def profile_from_args(profiles, settings, options, env, cwd, cache): diff --git a/conans/test/integration/environment/test_buildenv_profile.py b/conans/test/integration/environment/test_buildenv_profile.py new file mode 100644 index 00000000000..4b10ead5d84 --- /dev/null +++ b/conans/test/integration/environment/test_buildenv_profile.py @@ -0,0 +1,53 @@ +import textwrap + +import pytest + +from conans.test.utils.tools import TestClient + + +@pytest.fixture +def client(): + conanfile = textwrap.dedent(""" + from conans import ConanFile + class Pkg(ConanFile): + def generate(self): + for var in (1, 2): + v = self.buildenv.value("MyVar{}".format(var)) + self.output.info("MyVar{}={}!!".format(var, v)) + """) + profile1 = textwrap.dedent(""" + [buildenv] + MyVar1=MyValue1_1 + MyVar2=MyValue2_1 + """) + client = TestClient() + client.save({"conanfile.py": conanfile, + "profile1": profile1}) + return client + + +def test_buildenv_profile_cli(client): + profile2 = textwrap.dedent(""" + [buildenv] + MyVar1=MyValue1_2 + MyVar2+=MyValue2_2 + """) + client.save({"profile2": profile2}) + + client.run("install . -pr=profile1 -pr=profile2") + assert "conanfile.py: MyVar1=MyValue1_2!!" in client.out + assert "conanfile.py: MyVar2=MyValue2_1 MyValue2_2" in client.out + + +def test_buildenv_profile_include(client): + profile2 = textwrap.dedent(""" + include(profile1) + [buildenv] + MyVar1=MyValue1_2 + MyVar2+=MyValue2_2 + """) + client.save({"profile2": profile2}) + + client.run("install . -pr=profile2") + assert "conanfile.py: MyVar1=MyValue1_2!!" in client.out + assert "conanfile.py: MyVar2=MyValue2_1 MyValue2_2" in client.out From a96287c80146d926a4bf315ba01d6a8bcb9bda42 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 26 Feb 2021 00:49:28 +0100 Subject: [PATCH 07/28] minor interfaces --- conans/client/graph/graph_builder.py | 4 ++-- conans/model/conan_file.py | 6 ++++++ conans/model/conanfile_interface.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py index b33a74e60f1..1e1689a5397 100644 --- a/conans/client/graph/graph_builder.py +++ b/conans/client/graph/graph_builder.py @@ -53,7 +53,6 @@ def load_graph(self, root_node, check_updates, update, remotes, profile_host, pr initial = graph_lock.initial_counter if graph_lock else None dep_graph = DepsGraph(initial_node_id=initial) # compute the conanfile entry point for this dependency graph - root_node.conanfile.context = CONTEXT_HOST # FIXME: Is this always true? root_node.public_closure.add(root_node) root_node.public_deps.add(root_node) root_node.transitive_closure[root_node.name] = root_node @@ -206,7 +205,8 @@ def _expand_require(self, require, node, graph, check_updates, update, remotes, remotes, profile_host, profile_build, graph_lock, context_switch=context_switch, populate_settings_target=populate_settings_target) - new_node.conanfile._conan_context = context + # Information for consumers and new visitors + new_node.conanfile._conan_context = context # TODO: Improve this information passing node.conanfile.dependencies.add(new_node.conanfile) # The closure of a new node starts with just itself new_node.public_closure.add(new_node) diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index c8959729b92..39ef4ea84c0 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -151,11 +151,17 @@ def __init__(self, output, runner, display_name="", user=None, channel=None): self.buildenv_info = Environment() self.runenv_info = Environment() self._conan_buildenv = None # The profile buildenv, will be assigned initialize() + self._conan_context = None # build/host context + + @property + def context(self): + return self._conan_context @property def buildenv(self): # Lazy computation of the package buildenv based on the profileone if not isinstance(self._conan_buildenv, Environment): + # TODO: missing user/channel ref_str = "{}/{}".format(self.name, self.version) self._conan_buildenv = self._conan_buildenv.get_env(ref_str) return self._conan_buildenv diff --git a/conans/model/conanfile_interface.py b/conans/model/conanfile_interface.py index 194f634c4e4..d4e6bd5c11a 100644 --- a/conans/model/conanfile_interface.py +++ b/conans/model/conanfile_interface.py @@ -27,4 +27,4 @@ def settings(self): @property def context(self): - return self._conanfile._conan_context + return self._conanfile.context From 3d318e38feb66ae87ad0f8b96fa23743105fe057 Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 26 Feb 2021 21:37:34 +0100 Subject: [PATCH 08/28] fixing tests --- .../unittests/client/graph/deps_graph_test.py | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/conans/test/unittests/client/graph/deps_graph_test.py b/conans/test/unittests/client/graph/deps_graph_test.py index ac55cf40ee5..0ccba61d4b3 100644 --- a/conans/test/unittests/client/graph/deps_graph_test.py +++ b/conans/test/unittests/client/graph/deps_graph_test.py @@ -1,4 +1,5 @@ import unittest +from unittest.mock import Mock from conans.client.graph.graph import CONTEXT_HOST from conans.client.graph.graph_builder import DepsGraph, Node @@ -30,9 +31,9 @@ def test_basic_levels(self): ref3 = ConanFileReference.loads("Hello/3.0@user/stable") deps = DepsGraph() - n1 = Node(ref1, 1, context=CONTEXT_HOST) - n2 = Node(ref2, 2, context=CONTEXT_HOST) - n3 = Node(ref3, 3, context=CONTEXT_HOST) + n1 = Node(ref1, Mock(), context=CONTEXT_HOST) + n2 = Node(ref2, Mock(), context=CONTEXT_HOST) + n3 = Node(ref3, Mock(), context=CONTEXT_HOST) deps.add_node(n1) deps.add_node(n2) deps.add_node(n3) @@ -47,10 +48,10 @@ def test_multi_levels(self): ref32 = ConanFileReference.loads("Hello/32.0@user/stable") deps = DepsGraph() - n1 = Node(ref1, 1, context=CONTEXT_HOST) - n2 = Node(ref2, 2, context=CONTEXT_HOST) - n31 = Node(ref31, 31, context=CONTEXT_HOST) - n32 = Node(ref32, 32, context=CONTEXT_HOST) + n1 = Node(ref1, Mock(), context=CONTEXT_HOST) + n2 = Node(ref2, Mock(), context=CONTEXT_HOST) + n31 = Node(ref31, Mock(), context=CONTEXT_HOST) + n32 = Node(ref32, Mock(), context=CONTEXT_HOST) deps.add_node(n1) deps.add_node(n2) deps.add_node(n32) @@ -69,11 +70,11 @@ def test_multi_levels_2(self): ref32 = ConanFileReference.loads("Hello/32.0@user/stable") deps = DepsGraph() - n1 = Node(ref1, 1, context=CONTEXT_HOST) - n2 = Node(ref2, 2, context=CONTEXT_HOST) - n5 = Node(ref5, 5, context=CONTEXT_HOST) - n31 = Node(ref31, 31, context=CONTEXT_HOST) - n32 = Node(ref32, 32, context=CONTEXT_HOST) + n1 = Node(ref1, Mock(), context=CONTEXT_HOST) + n2 = Node(ref2, Mock(), context=CONTEXT_HOST) + n5 = Node(ref5, Mock(), context=CONTEXT_HOST) + n31 = Node(ref31, Mock(), context=CONTEXT_HOST) + n32 = Node(ref32, Mock(), context=CONTEXT_HOST) deps.add_node(n1) deps.add_node(n5) deps.add_node(n2) @@ -94,11 +95,11 @@ def test_multi_levels_3(self): ref32 = ConanFileReference.loads("Hello/32.0@user/stable") deps = DepsGraph() - n1 = Node(ref1, 1, context=CONTEXT_HOST) - n2 = Node(ref2, 2, context=CONTEXT_HOST) - n5 = Node(ref5, 5, context=CONTEXT_HOST) - n31 = Node(ref31, 31, context=CONTEXT_HOST) - n32 = Node(ref32, 32, context=CONTEXT_HOST) + n1 = Node(ref1, Mock(), context=CONTEXT_HOST) + n2 = Node(ref2, Mock(), context=CONTEXT_HOST) + n5 = Node(ref5, Mock(), context=CONTEXT_HOST) + n31 = Node(ref31, Mock(), context=CONTEXT_HOST) + n32 = Node(ref32, Mock(), context=CONTEXT_HOST) deps.add_node(n1) deps.add_node(n5) deps.add_node(n2) From 247f98c043719d192590837a735e9884046dc73f Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 26 Feb 2021 21:44:01 +0100 Subject: [PATCH 09/28] fix py27 test --- conans/test/unittests/client/graph/deps_graph_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/test/unittests/client/graph/deps_graph_test.py b/conans/test/unittests/client/graph/deps_graph_test.py index 0ccba61d4b3..4e9a3757fb0 100644 --- a/conans/test/unittests/client/graph/deps_graph_test.py +++ b/conans/test/unittests/client/graph/deps_graph_test.py @@ -1,5 +1,5 @@ import unittest -from unittest.mock import Mock +from mock import Mock from conans.client.graph.graph import CONTEXT_HOST from conans.client.graph.graph_builder import DepsGraph, Node From 997936bcfd5f8ca9d205784b7c0aa19491658120 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 1 Mar 2021 11:13:20 +0100 Subject: [PATCH 10/28] trying with shell only, not bash --- conan/tools/env/environment.py | 17 +++++++++++------ conans/test/unittests/tools/env/test_env.py | 8 ++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index b4743c78074..424c830cb53 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -23,7 +23,8 @@ def environment_wrap_command(filename, cmd): if filename.endswith(".bat"): return "{} && {}".format(filename, cmd) elif filename.endswith(".sh"): - return 'bash -c ". {} && {}"'.format(filename, cmd.replace('"', r'\"')) + # Generic shell, not bash specific, but deactivate will not work + return '. ./{} && {}'.format(filename, cmd.replace('"', r'\"')) raise Exception("Unsupported environment file type {}".format(filename)) @@ -97,7 +98,7 @@ def prepend_path(self, name, value): value = self._list_value(value, _PathSep) self._values[name] = value + [_PathSep] + [_EnvVarPlaceHolder] - def save_bat(self, filename, generate_deactivate=True, pathsep=os.pathsep): + def save_bat(self, filename, generate_deactivate=False, pathsep=os.pathsep): deactivate = textwrap.dedent("""\ echo Capturing current environment in deactivate_{filename} setlocal @@ -131,7 +132,7 @@ def save_bat(self, filename, generate_deactivate=True, pathsep=os.pathsep): content = "\n".join(result) save(filename, content) - def save_ps1(self, filename, generate_deactivate=True, pathsep=os.pathsep): + def save_ps1(self, filename, generate_deactivate=False, pathsep=os.pathsep): # FIXME: This is broken and doesnt work deactivate = "" capture = textwrap.dedent("""\ @@ -146,13 +147,13 @@ def save_ps1(self, filename, generate_deactivate=True, pathsep=os.pathsep): content = "\n".join(result) save(filename, content) - def save_sh(self, filename, pathsep=os.pathsep): - capture = textwrap.dedent("""\ + def save_sh(self, filename, generate_deactivate=False, pathsep=os.pathsep): + deactivate = textwrap.dedent("""\ echo Capturing current environment in deactivate_{filename} echo echo Restoring variables >> deactivate_{filename} for v in {vars} do - value=${{!v}} + value=$(printenv $v) if [ -n "$value" ] then echo export "$v=$value" >> deactivate_{filename} @@ -162,6 +163,10 @@ def save_sh(self, filename, pathsep=os.pathsep): done echo Configuring environment variables """.format(filename=filename, vars=" ".join(self._values.keys()))) + capture = textwrap.dedent("""\ + {deactivate} + echo Configuring environment variables + """).format(deactivate=deactivate if generate_deactivate else "") result = [capture] for varname, varvalues in self._values.items(): value = self._format_value(varname, varvalues, "${name}", pathsep) diff --git a/conans/test/unittests/tools/env/test_env.py b/conans/test/unittests/tools/env/test_env.py index a7ded847d92..51399bc5786 100644 --- a/conans/test/unittests/tools/env/test_env.py +++ b/conans/test/unittests/tools/env/test_env.py @@ -236,7 +236,7 @@ def check(cmd_): with chdir(folder): if platform.system() == "Windows": - env.save_bat("test.bat", pathsep=":") + env.save_bat("test.bat", pathsep=":", generate_deactivate=True) save("display.bat", display_bat) cmd = "test.bat && display.bat && deactivate_test.bat && display.bat" check(cmd) @@ -246,10 +246,10 @@ def check(cmd_): # cmd = r'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe' # check(cmd) else: - env.save_sh("test.sh") + env.save_sh("test.sh", generate_deactivate=True) save("display.sh", display_sh) os.chmod("display.sh", 0o777) - cmd = 'bash -c ". test.sh && ./display.sh && . deactivate_test.sh && ./display.sh"' + cmd = '. ./test.sh && ./display.sh && . ./deactivate_test.sh && ./display.sh' check(cmd) @@ -270,7 +270,7 @@ def test_windows_case_insensitive(): """) with chdir(folder): - env.save_bat("test.bat") + env.save_bat("test.bat", generate_deactivate=True) save("display.bat", display_bat) cmd = "test.bat && display.bat && deactivate_test.bat && display.bat" out, _ = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, From fa0ca3da99bfdff8692438579477df09694fae7b Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 1 Mar 2021 11:37:44 +0100 Subject: [PATCH 11/28] not necessary to escape quotes --- conan/tools/env/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index 424c830cb53..470ad718b43 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -24,7 +24,7 @@ def environment_wrap_command(filename, cmd): return "{} && {}".format(filename, cmd) elif filename.endswith(".sh"): # Generic shell, not bash specific, but deactivate will not work - return '. ./{} && {}'.format(filename, cmd.replace('"', r'\"')) + return '. ./{} && {}'.format(filename, cmd) raise Exception("Unsupported environment file type {}".format(filename)) From 1a0abdf3c15d86859c9586944f59f73c1eaeba4d Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 2 Mar 2021 00:52:24 +0100 Subject: [PATCH 12/28] merged develop --- conan/tools/env/environment.py | 1 - conans/client/profile_loader.py | 1 - conans/test/unittests/tools/env/test_env.py | 6 ++++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index 470ad718b43..20ddccaf863 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -141,7 +141,6 @@ def save_ps1(self, filename, generate_deactivate=False, pathsep=os.pathsep): result = [capture] for varname, varvalues in self._values.items(): value = self._format_value(varname, varvalues, "$env:{name}", pathsep) - result.append('Write-Output "Error: whatever message {}"'.format(varname)) result.append('$env:{}={}'.format(varname, value)) content = "\n".join(result) diff --git a/conans/client/profile_loader.py b/conans/client/profile_loader.py index e3c0b7f50cd..0b67d014b57 100644 --- a/conans/client/profile_loader.py +++ b/conans/client/profile_loader.py @@ -229,7 +229,6 @@ def get_package_name_value(item): if doc.buildenv: buildenv = ProfileEnvironment() buildenv.loads(doc.buildenv) - # TODO: compose ProfileEnvironment base_profile.buildenv.compose(buildenv) diff --git a/conans/test/unittests/tools/env/test_env.py b/conans/test/unittests/tools/env/test_env.py index 51399bc5786..3704483e584 100644 --- a/conans/test/unittests/tools/env/test_env.py +++ b/conans/test/unittests/tools/env/test_env.py @@ -242,8 +242,10 @@ def check(cmd_): check(cmd) # FIXME: Powershell still not working # env.save_ps1("test.ps1", pathsep=":") - # print(load("test.ps1")) - # cmd = r'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe' + # cmd = 'powershell.exe -ExecutionPolicy ./test.ps1; gci env:' + # shell = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # (stdout, stderr) = shell.communicate() + # stdout, stderr = decode_text(stdout), decode_text(stderr) # check(cmd) else: env.save_sh("test.sh", generate_deactivate=True) From 1f66a6c964a62cca71ebd650ce8674035598ffe5 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 3 Mar 2021 10:34:59 +0100 Subject: [PATCH 13/28] poc for self.run() env definition --- conan/tools/env/environment.py | 28 ++++++++++---- conans/model/conan_file.py | 6 ++- conans/test/functional/toolchains/test_env.py | 38 +++++++++++++------ 3 files changed, 52 insertions(+), 20 deletions(-) diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index 20ddccaf863..d52c8b7cdf2 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -19,13 +19,27 @@ class _PathSep: pass -def environment_wrap_command(filename, cmd): - if filename.endswith(".bat"): - return "{} && {}".format(filename, cmd) - elif filename.endswith(".sh"): - # Generic shell, not bash specific, but deactivate will not work - return '. ./{} && {}'.format(filename, cmd) - raise Exception("Unsupported environment file type {}".format(filename)) +def environment_wrap_command(filename, cmd, cwd=None): + assert filename + filenames = [filename] if not isinstance(filename, list) else filename + bats, shs = [], [] + for f in filenames: + full_path = os.path.join(cwd, f) if cwd else f + if os.path.isfile("{}.bat".format(full_path)): + bats.append(f) + elif os.path.isfile("{}.sh".format(full_path)): + shs.append(f) + if bats and shs: + raise ConanException("Cannot wrap command with different envs, {} - {}".format(bats, shs)) + + if bats: + command = " && ".join(bats) + return "{} && {}".format(command, cmd) + elif shs: + command = " && ".join(". ./{}".format(f) for f in shs) + return "{} && {}".format(command, cmd) + else: + return cmd class Environment: diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index 9eebbae1c83..584e48bd9f0 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -5,6 +5,7 @@ from six import string_types from conan.tools.env import Environment +from conan.tools.env.environment import environment_wrap_command from conans.client import tools from conans.client.graph.conanfile_dependencies import ConanFileDependencies from conans.client.output import ScopedOutput @@ -323,7 +324,10 @@ def package_info(self): """ def run(self, command, output=True, cwd=None, win_bash=False, subsystem=None, msys_mingw=True, - ignore_errors=False, run_environment=False, with_login=True): + ignore_errors=False, run_environment=False, with_login=True, env="buildenv"): + + command = environment_wrap_command(env, command) + def _run(): if not win_bash: return self._conan_runner(command, output, os.path.abspath(RUN_LOG_NAME), cwd) diff --git a/conans/test/functional/toolchains/test_env.py b/conans/test/functional/toolchains/test_env.py index 70195a166db..164b0faa8b7 100644 --- a/conans/test/functional/toolchains/test_env.py +++ b/conans/test/functional/toolchains/test_env.py @@ -80,33 +80,46 @@ def package_info(self): def test_complete(client): conanfile = textwrap.dedent(""" - from conans import ConanFile - class Pkg(ConanFile): - generators = "VirtualEnv" - requires = "openssl/1.0" - build_requires = "mycmake/1.0" - - def build_requirements(self): - self.build_requires("mygtest/1.0", force_host_context=True) + import platform + from conans import ConanFile + class Pkg(ConanFile): + generators = "VirtualEnv" + requires = "openssl/1.0" + build_requires = "mycmake/1.0" + + def build_requirements(self): + self.build_requires("mygtest/1.0", force_host_context=True) + + def build(self): + mybuild_cmd = "mycmake.bat" if platform.system() == "Windows" else "mycmake.sh" + self.run(mybuild_cmd) + mytest_cmd = "mygtest.bat" if platform.system() == "Windows" else "mygtest.sh" + self.run(mytest_cmd, env="runenv") """) client.save({"conanfile.py": conanfile}) client.run("install . -s:b os=Windows -s:h os=Linux --build=missing") # Run the BUILD environment ext = "bat" if platform.system() == "Windows" else "sh" # TODO: Decide on logic .bat vs .sh - cmd = environment_wrap_command("buildenv.{}".format(ext), "mycmake.{}".format(ext)) + cmd = environment_wrap_command("buildenv", "mycmake.{}".format(ext), cwd=client.current_folder) client.run_command(cmd) assert "MYCMAKE=Windows!!" in client.out assert "MYOPENSSL=Windows!!" in client.out # Run the RUN environment - cmd = environment_wrap_command("runenv.{}".format(ext), + cmd = environment_wrap_command("runenv", "mygtest.{ext} && .{sep}myrunner.{ext}".format(ext=ext, - sep=os.sep)) + sep=os.sep), + cwd=client.current_folder) client.run_command(cmd) assert "MYGTEST=Linux!!" in client.out assert "MYGTESTVAR=MyGTestValueLinux!!" in client.out + client.run("build .") + assert "MYCMAKE=Windows!!" in client.out + assert "MYOPENSSL=Windows!!" in client.out + assert "MYGTEST=Linux!!" in client.out + def test_profile_buildenv(client): conanfile = GenConanfile().with_generator("VirtualEnv") @@ -137,7 +150,8 @@ def test_profile_buildenv(client): client.run("install . -pr=myprofile") # Run the BUILD environment ext = "bat" if platform.system() == "Windows" else "sh" # TODO: Decide on logic .bat vs .sh - cmd = environment_wrap_command("buildenv.{}".format(ext), "mycompiler.{}".format(ext)) + cmd = environment_wrap_command("buildenv", "mycompiler.{}".format(ext), + cwd=client.current_folder) client.run_command(cmd) assert "MYCOMPILER!!" in client.out assert "MYPATH=" in client.out From f78169a45da5b367216a4a08ba0fd565abad83ab Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 3 Mar 2021 11:03:31 +0100 Subject: [PATCH 14/28] fix test --- conan/tools/env/environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index d52c8b7cdf2..b4308f704ec 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -26,9 +26,9 @@ def environment_wrap_command(filename, cmd, cwd=None): for f in filenames: full_path = os.path.join(cwd, f) if cwd else f if os.path.isfile("{}.bat".format(full_path)): - bats.append(f) + bats.append("{}.bat".format(f)) elif os.path.isfile("{}.sh".format(full_path)): - shs.append(f) + shs.append("{}.sh".format(f)) if bats and shs: raise ConanException("Cannot wrap command with different envs, {} - {}".format(bats, shs)) From b6dcb0715b70884f9477b11abb1b00579b145306 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 3 Mar 2021 11:59:22 +0100 Subject: [PATCH 15/28] renaming files --- conan/tools/cmake/cmake.py | 11 ----------- conan/tools/env/virtualenv.py | 8 ++++---- conans/model/conan_file.py | 2 +- conans/test/functional/toolchains/test_env.py | 9 +++++---- 4 files changed, 10 insertions(+), 20 deletions(-) diff --git a/conan/tools/cmake/cmake.py b/conan/tools/cmake/cmake.py index 9dfc3cab244..0faec37ad65 100644 --- a/conan/tools/cmake/cmake.py +++ b/conan/tools/cmake/cmake.py @@ -3,7 +3,6 @@ from conan.tools.cmake.base import CMakeToolchainBase from conan.tools.cmake.utils import get_generator, is_multi_configuration -from conan.tools.env.environment import environment_wrap_command from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg from conans.client import tools from conans.client.build import join_arguments @@ -83,11 +82,6 @@ def configure(self, source_folder=None): generator = '-G "{}" '.format(self._generator) if self._generator else "" command = "%s %s%s" % (self._cmake_program, generator, arg_list) - # Need to activate the buildenv if existing - env_filename = "buildenv.bat" if platform.system() == "Windows" else "buildenv.sh" - if os.path.isfile(env_filename): - command = environment_wrap_command(env_filename, command) - is_windows_mingw = platform.system() == "Windows" and self._generator == "MinGW Makefiles" self._conanfile.output.info("CMake command: %s" % command) with chdir(build_folder): @@ -126,11 +120,6 @@ def _build(self, build_type=None, target=None): arg_list = [args_to_string([bf]), build_config, args_to_string(args)] command = "%s --build %s" % (self._cmake_program, join_arguments(arg_list)) - # Need to activate the buildenv if existing - env_filename = "buildenv.bat" if platform.system() == "Windows" else "buildenv.sh" - if os.path.isfile(env_filename): - command = environment_wrap_command(env_filename, command) - self._conanfile.output.info("CMake command: %s" % command) self._conanfile.run(command) diff --git a/conan/tools/env/virtualenv.py b/conan/tools/env/virtualenv.py index 79135dd120c..fc6f24be943 100644 --- a/conan/tools/env/virtualenv.py +++ b/conan/tools/env/virtualenv.py @@ -91,8 +91,8 @@ def generate(self): # FIXME: Use settings, not platform Not always defined :( # os_ = self._conanfile.settings_build.get_safe("os") if platform.system() == "Windows": - build_env.save_bat("buildenv.bat") - run_env.save_bat("runenv.bat") + build_env.save_bat("conanbuildenv.bat") + run_env.save_bat("conanrunenv.bat") else: - build_env.save_sh("buildenv.sh") - run_env.save_sh("runenv.sh") + build_env.save_sh("conanbuildenv.sh") + run_env.save_sh("conanrunenv.sh") diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index 584e48bd9f0..b3d743f2914 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -324,7 +324,7 @@ def package_info(self): """ def run(self, command, output=True, cwd=None, win_bash=False, subsystem=None, msys_mingw=True, - ignore_errors=False, run_environment=False, with_login=True, env="buildenv"): + ignore_errors=False, run_environment=False, with_login=True, env="conanbuildenv"): command = environment_wrap_command(env, command) diff --git a/conans/test/functional/toolchains/test_env.py b/conans/test/functional/toolchains/test_env.py index 164b0faa8b7..46285f630fe 100644 --- a/conans/test/functional/toolchains/test_env.py +++ b/conans/test/functional/toolchains/test_env.py @@ -94,20 +94,21 @@ def build(self): mybuild_cmd = "mycmake.bat" if platform.system() == "Windows" else "mycmake.sh" self.run(mybuild_cmd) mytest_cmd = "mygtest.bat" if platform.system() == "Windows" else "mygtest.sh" - self.run(mytest_cmd, env="runenv") + self.run(mytest_cmd, env="conanrunenv") """) client.save({"conanfile.py": conanfile}) client.run("install . -s:b os=Windows -s:h os=Linux --build=missing") # Run the BUILD environment ext = "bat" if platform.system() == "Windows" else "sh" # TODO: Decide on logic .bat vs .sh - cmd = environment_wrap_command("buildenv", "mycmake.{}".format(ext), cwd=client.current_folder) + cmd = environment_wrap_command("conanbuildenv", "mycmake.{}".format(ext), + cwd=client.current_folder) client.run_command(cmd) assert "MYCMAKE=Windows!!" in client.out assert "MYOPENSSL=Windows!!" in client.out # Run the RUN environment - cmd = environment_wrap_command("runenv", + cmd = environment_wrap_command("conanrunenv", "mygtest.{ext} && .{sep}myrunner.{ext}".format(ext=ext, sep=os.sep), cwd=client.current_folder) @@ -150,7 +151,7 @@ def test_profile_buildenv(client): client.run("install . -pr=myprofile") # Run the BUILD environment ext = "bat" if platform.system() == "Windows" else "sh" # TODO: Decide on logic .bat vs .sh - cmd = environment_wrap_command("buildenv", "mycompiler.{}".format(ext), + cmd = environment_wrap_command("conanbuildenv", "mycompiler.{}".format(ext), cwd=client.current_folder) client.run_command(cmd) assert "MYCOMPILER!!" in client.out From e3a3cf065e55e3bebb48874ab3d798ee11301898 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 4 Mar 2021 14:18:16 +0100 Subject: [PATCH 16/28] change default to opt-out --- conan/tools/env/virtualenv.py | 1 + conans/client/generators/__init__.py | 8 ++++++++ conans/model/conan_file.py | 1 + conans/test/functional/toolchains/test_env.py | 3 +-- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/conan/tools/env/virtualenv.py b/conan/tools/env/virtualenv.py index fc6f24be943..6c3460c4aae 100644 --- a/conan/tools/env/virtualenv.py +++ b/conan/tools/env/virtualenv.py @@ -11,6 +11,7 @@ class VirtualEnv: def __init__(self, conanfile): self._conanfile = conanfile + self._conanfile.virtualenv = False def build_environment(self): """ collects the buildtime information from dependencies. This is the typical use case diff --git a/conans/client/generators/__init__.py b/conans/client/generators/__init__.py index e002a6df2d2..497489b436b 100644 --- a/conans/client/generators/__init__.py +++ b/conans/client/generators/__init__.py @@ -1,3 +1,5 @@ +import os +import platform import traceback from os.path import join @@ -203,3 +205,9 @@ def write_toolchain(conanfile, path, output): with chdir(path): with conanfile_exception_formatter(str(conanfile), "generate"): conanfile.generate() + + if conanfile.virtualenv: + with chdir(path): + from conan.tools.env.virtualenv import VirtualEnv + env = VirtualEnv(conanfile) + env.generate() diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index b3d743f2914..3e9bd9804e5 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -154,6 +154,7 @@ def __init__(self, output, runner, display_name="", user=None, channel=None, req self.runenv_info = Environment() self._conan_buildenv = None # The profile buildenv, will be assigned initialize() self._conan_node = None # access to container Node object, to access info, context, deps... + self.virtualenv = True # Set to false to opt-out automatic usage of VirtualEnv @property def context(self): diff --git a/conans/test/functional/toolchains/test_env.py b/conans/test/functional/toolchains/test_env.py index 46285f630fe..886efec9e6a 100644 --- a/conans/test/functional/toolchains/test_env.py +++ b/conans/test/functional/toolchains/test_env.py @@ -83,7 +83,6 @@ def test_complete(client): import platform from conans import ConanFile class Pkg(ConanFile): - generators = "VirtualEnv" requires = "openssl/1.0" build_requires = "mycmake/1.0" @@ -123,7 +122,7 @@ def build(self): def test_profile_buildenv(client): - conanfile = GenConanfile().with_generator("VirtualEnv") + conanfile = GenConanfile() # Some scripts in a random system folders, path adding to the profile [env] compiler_bat = "@echo off\necho MYCOMPILER!!\necho MYPATH=%PATH%" From 14829236330f80bff480af99c5885f3f8d6a04e8 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 4 Mar 2021 17:13:42 +0100 Subject: [PATCH 17/28] minor changes --- conan/tools/env/environment.py | 5 +++++ conan/tools/env/virtualenv.py | 15 +++++++++------ .../integration/generators/xcode_gcc_vs_test.py | 1 - 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index b4308f704ec..d606c58ab3d 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -48,6 +48,11 @@ def __init__(self): # It being ordered allows for Windows case-insensitive composition self._values = OrderedDict() # {var_name: [] of values, including separators} + def __bool__(self): + return bool(self._values) + + __nonzero__ = __bool__ + def __repr__(self): return repr(self._values) diff --git a/conan/tools/env/virtualenv.py b/conan/tools/env/virtualenv.py index 6c3460c4aae..3915c05ab06 100644 --- a/conan/tools/env/virtualenv.py +++ b/conan/tools/env/virtualenv.py @@ -55,6 +55,8 @@ def _runenv_from_cpp_info(conanfile_dep): """ dyn_runenv = Environment() cpp_info = conanfile_dep.cpp_info + if cpp_info is None: # This happens when the dependency is a private one = BINARY_SKIP + return dyn_runenv if cpp_info.exes: dyn_runenv.prepend_path("PATH", cpp_info.bin_paths) # If it is a build_require this will be the build-os, otherwise it will be the host-os @@ -91,9 +93,10 @@ def generate(self): run_env = self.run_environment() # FIXME: Use settings, not platform Not always defined :( # os_ = self._conanfile.settings_build.get_safe("os") - if platform.system() == "Windows": - build_env.save_bat("conanbuildenv.bat") - run_env.save_bat("conanrunenv.bat") - else: - build_env.save_sh("conanbuildenv.sh") - run_env.save_sh("conanrunenv.sh") + if build_env: # Only if there is something defined + if platform.system() == "Windows": + build_env.save_bat("conanbuildenv.bat") + run_env.save_bat("conanrunenv.bat") + else: + build_env.save_sh("conanbuildenv.sh") + run_env.save_sh("conanrunenv.sh") diff --git a/conans/test/integration/generators/xcode_gcc_vs_test.py b/conans/test/integration/generators/xcode_gcc_vs_test.py index 5c98c5de4da..29c6026a1f4 100644 --- a/conans/test/integration/generators/xcode_gcc_vs_test.py +++ b/conans/test/integration/generators/xcode_gcc_vs_test.py @@ -8,7 +8,6 @@ from conans.paths import (BUILD_INFO, BUILD_INFO_CMAKE, BUILD_INFO_GCC, BUILD_INFO_VISUAL_STUDIO, BUILD_INFO_XCODE, CONANFILE_TXT, CONANINFO) from conans.test.utils.tools import TestClient -from conans.util.files import load class VSXCodeGeneratorsTest(unittest.TestCase): From 935b3d0c520acefb633b67d551e68a3d6bea5ee6 Mon Sep 17 00:00:00 2001 From: memsharded Date: Sat, 6 Mar 2021 01:27:33 +0100 Subject: [PATCH 18/28] more complete real test --- .../toolchains/cmake/test_cmakegen.py | 1 + .../functional/toolchains/env/__init__.py | 0 .../toolchains/env/test_complete.py | 101 ++++++++++++++++++ .../environment}/test_env.py | 1 + 4 files changed, 103 insertions(+) create mode 100644 conans/test/functional/toolchains/env/__init__.py create mode 100644 conans/test/functional/toolchains/env/test_complete.py rename conans/test/{functional/toolchains => integration/environment}/test_env.py (99%) diff --git a/conans/test/functional/toolchains/cmake/test_cmakegen.py b/conans/test/functional/toolchains/cmake/test_cmakegen.py index df4be960e2a..50228b4ca5d 100644 --- a/conans/test/functional/toolchains/cmake/test_cmakegen.py +++ b/conans/test/functional/toolchains/cmake/test_cmakegen.py @@ -33,6 +33,7 @@ class App(ConanFile): requires = "hello/0.1" build_requires = "cmakewrapper/0.1" generators = "CMakeGen" + apply_env = False def build(self): cmake = CMake(self) diff --git a/conans/test/functional/toolchains/env/__init__.py b/conans/test/functional/toolchains/env/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/conans/test/functional/toolchains/env/test_complete.py b/conans/test/functional/toolchains/env/test_complete.py new file mode 100644 index 00000000000..7d4533a4878 --- /dev/null +++ b/conans/test/functional/toolchains/env/test_complete.py @@ -0,0 +1,101 @@ +import textwrap + +from conans.test.assets.sources import gen_function_cpp +from conans.test.utils.tools import TestClient + + +def test_complete(): + client = TestClient() + client.run("new myopenssl/1.0 -m=v2_cmake") + client.run("create . -o myopenssl:shared=True") + client.run("create . -o myopenssl:shared=True -s build_type=Debug") + + mycmake_main = gen_function_cpp(name="main", msg="mycmake", + includes=["myopenssl"], calls=["myopenssl"]) + mycmake_conanfile = textwrap.dedent(""" + from conans import ConanFile + from conan.tools.cmake import CMake + class App(ConanFile): + settings = "os", "arch", "compiler", "build_type" + requires = "myopenssl/1.0" + default_options = {"myopenssl:shared": True} + generators = "CMakeGen" + exports = "*" + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + src = str(self.settings.build_type) if self.settings.os == "Windows" else None + self.copy("mycmake*", src=src, dst="bin") + + def package_info(self): + self.cpp_info.exes = ["mycmake"] + self.cpp_info.bindirs = ["bin"] + """) + mycmake_cmakelists = textwrap.dedent(""" + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) + cmake_minimum_required(VERSION 3.15) + project(MyCmake CXX) + + find_package(myopenssl REQUIRED) + add_executable(mycmake main.cpp) + target_link_libraries(mycmake PRIVATE myopenssl::myopenssl) + """) + client.save({"conanfile.py": mycmake_conanfile, + "CMakeLists.txt": mycmake_cmakelists, + "main.cpp": mycmake_main}, clean_first=True) + client.run("create . mycmake/1.0@") + + mylib = textwrap.dedent(r""" + from conans import ConanFile + import os + from conan.tools.cmake import CMake + class Pkg(ConanFile): + settings = "os", "compiler", "build_type", "arch" + build_requires = "mycmake/1.0" + requires = "myopenssl/1.0" + default_options = {"myopenssl:shared": True} + exports_sources = "CMakeLists.txt", "main.cpp" + generators = "CMakeGen" + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + self.run("mycmake") + self.output.info("RUNNING MYAPP") + if self.settings.os == "Windows": + self.run(os.sep.join([".", str(self.settings.build_type), "myapp"]), + env="conanrunenv") + else: + self.run(os.sep.join([".", "myapp"]), env="conanrunenv") + """) + + cmakelists = textwrap.dedent(""" + set(CMAKE_CXX_COMPILER_WORKS 1) + set(CMAKE_CXX_ABI_COMPILED 1) + cmake_minimum_required(VERSION 3.15) + project(MyApp CXX) + + find_package(myopenssl) + add_executable(myapp main.cpp) + target_link_libraries(myapp myopenssl::myopenssl) + """) + + client.save({"conanfile.py": mylib, + "main.cpp": gen_function_cpp(name="main", msg="myapp", includes=["myopenssl"], + calls=["myopenssl"]), + "CMakeLists.txt": cmakelists}, + clean_first=True) + + client.run("create . myapp/0.1@ -s:b build_type=Release -s:h build_type=Debug") + first, last = str(client.out).split("RUNNING MYAPP") + assert "mycmake: Release!" in first + assert "myopenssl/1.0: Hello World Release!" in first + + assert "myapp: Debug!" in last + assert "myopenssl/1.0: Hello World Debug!" in last diff --git a/conans/test/functional/toolchains/test_env.py b/conans/test/integration/environment/test_env.py similarity index 99% rename from conans/test/functional/toolchains/test_env.py rename to conans/test/integration/environment/test_env.py index 886efec9e6a..d9e29ab769a 100644 --- a/conans/test/functional/toolchains/test_env.py +++ b/conans/test/integration/environment/test_env.py @@ -85,6 +85,7 @@ def test_complete(client): class Pkg(ConanFile): requires = "openssl/1.0" build_requires = "mycmake/1.0" + apply_env = False def build_requirements(self): self.build_requires("mygtest/1.0", force_host_context=True) From 15dbf07055832d679efada3aaefdae877620d62f Mon Sep 17 00:00:00 2001 From: memsharded Date: Sun, 7 Mar 2021 23:22:32 +0100 Subject: [PATCH 19/28] fix test --- conans/test/functional/toolchains/env/test_complete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/test/functional/toolchains/env/test_complete.py b/conans/test/functional/toolchains/env/test_complete.py index 7d4533a4878..4b8aea0bdd3 100644 --- a/conans/test/functional/toolchains/env/test_complete.py +++ b/conans/test/functional/toolchains/env/test_complete.py @@ -28,7 +28,7 @@ def build(self): cmake.build() def package(self): - src = str(self.settings.build_type) if self.settings.os == "Windows" else None + src = str(self.settings.build_type) if self.settings.os == "Windows" else "" self.copy("mycmake*", src=src, dst="bin") def package_info(self): From 4221d32522736cd6f2377a3f4eaf15f345d8a465 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 8 Mar 2021 10:07:10 +0100 Subject: [PATCH 20/28] fix v2_cmake template for shared libs in OSX with run_environment=True --- conans/assets/templates/new_v2_cmake.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conans/assets/templates/new_v2_cmake.py b/conans/assets/templates/new_v2_cmake.py index a6ef8f799ca..52233e8e296 100644 --- a/conans/assets/templates/new_v2_cmake.py +++ b/conans/assets/templates/new_v2_cmake.py @@ -70,7 +70,8 @@ def imports(self): def test(self): if not tools.cross_building(self): os.chdir("bin") - self.run(".%sexample" % os.sep) + # TODO: The run_environment=True will be replaced by new Environment + self.run(".%sexample" % os.sep, run_environment=True) """ From e1bf328335fa576be2a4b7ef18ba32840499ebec Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 9 Mar 2021 18:34:10 +0100 Subject: [PATCH 21/28] improved v2_cmake new template with environment --- conan/tools/cmake/cmakegen.py | 7 +++++++ conan/tools/env/virtualenv.py | 5 ++++- conans/assets/templates/new_v2_cmake.py | 21 +++++-------------- .../toolchains/env/test_complete.py | 1 + 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/conan/tools/cmake/cmakegen.py b/conan/tools/cmake/cmakegen.py index 98859b9f8cc..628b816ef06 100644 --- a/conan/tools/cmake/cmakegen.py +++ b/conan/tools/cmake/cmakegen.py @@ -12,3 +12,10 @@ def generate(self): self.toolchain.generate() self.deps.generate() self.env.generate() + + def _output_path(self, value): + self.toolchain.output_path = value + self.deps.output_path = value + self.env.output_path = value + + output_path = property(fset=_output_path) # now value has only a setter diff --git a/conan/tools/env/virtualenv.py b/conan/tools/env/virtualenv.py index 3915c05ab06..5d836682a9b 100644 --- a/conan/tools/env/virtualenv.py +++ b/conan/tools/env/virtualenv.py @@ -96,7 +96,10 @@ def generate(self): if build_env: # Only if there is something defined if platform.system() == "Windows": build_env.save_bat("conanbuildenv.bat") - run_env.save_bat("conanrunenv.bat") else: build_env.save_sh("conanbuildenv.sh") + if run_env: + if platform.system() == "Windows": + run_env.save_bat("conanrunenv.bat") + else: run_env.save_sh("conanrunenv.sh") diff --git a/conans/assets/templates/new_v2_cmake.py b/conans/assets/templates/new_v2_cmake.py index 52233e8e296..0dfb7ef8c20 100644 --- a/conans/assets/templates/new_v2_cmake.py +++ b/conans/assets/templates/new_v2_cmake.py @@ -45,33 +45,22 @@ def package_info(self): test_conanfile_v2 = """import os from conans import ConanFile, tools -from conan.tools.cmake import CMakeToolchain, CMake, CMakeDeps +from conan.tools.cmake import CMake class {package_name}TestConan(ConanFile): settings = "os", "compiler", "build_type", "arch" - - def generate(self): - deps = CMakeDeps(self) - deps.generate() - tc = CMakeToolchain(self) - tc.generate() + generators = "CMakeGen" + apply_env = False def build(self): cmake = CMake(self) cmake.configure() cmake.build() - def imports(self): - self.copy("*.dll", dst="bin", src="bin") - self.copy("*.dylib*", dst="bin", src="lib") - self.copy('*.so*', dst='bin', src='lib') - def test(self): if not tools.cross_building(self): - os.chdir("bin") - # TODO: The run_environment=True will be replaced by new Environment - self.run(".%sexample" % os.sep, run_environment=True) + self.run(os.path.sep.join([".", "bin", "example"]), env="conanrunenv") """ @@ -85,7 +74,7 @@ def test(self): set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_MINSIZEREL ${{CMAKE_RUNTIME_OUTPUT_DIRECTORY}}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${{CMAKE_RUNTIME_OUTPUT_DIRECTORY}}) -find_package({name}) +find_package({name} CONFIG REQUIRED) add_executable(example example.cpp) target_link_libraries(example {name}::{name}) diff --git a/conans/test/functional/toolchains/env/test_complete.py b/conans/test/functional/toolchains/env/test_complete.py index 4b8aea0bdd3..986a8b53130 100644 --- a/conans/test/functional/toolchains/env/test_complete.py +++ b/conans/test/functional/toolchains/env/test_complete.py @@ -21,6 +21,7 @@ class App(ConanFile): default_options = {"myopenssl:shared": True} generators = "CMakeGen" exports = "*" + apply_env = False def build(self): cmake = CMake(self) From c43ed63561c793180b54b44090bebb5272f5885e Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 9 Mar 2021 18:57:33 +0100 Subject: [PATCH 22/28] cpp_info.exes do not exist yet --- conan/tools/env/virtualenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/tools/env/virtualenv.py b/conan/tools/env/virtualenv.py index 5d836682a9b..fe6e86c0991 100644 --- a/conan/tools/env/virtualenv.py +++ b/conan/tools/env/virtualenv.py @@ -57,7 +57,7 @@ def _runenv_from_cpp_info(conanfile_dep): cpp_info = conanfile_dep.cpp_info if cpp_info is None: # This happens when the dependency is a private one = BINARY_SKIP return dyn_runenv - if cpp_info.exes: + if cpp_info.bin_paths: # cpp_info.exes is not defined yet dyn_runenv.prepend_path("PATH", cpp_info.bin_paths) # If it is a build_require this will be the build-os, otherwise it will be the host-os os_ = conanfile_dep.settings.get_safe("os") From 3198d929638407ef0d308c6b9ae4a96d644184ae Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 9 Mar 2021 19:55:39 +0100 Subject: [PATCH 23/28] fix tests --- conan/tools/cmake/cmakegen.py | 2 +- conans/test/functional/toolchains/env/test_complete.py | 1 - conans/test/integration/command/new_test.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/conan/tools/cmake/cmakegen.py b/conan/tools/cmake/cmakegen.py index 628b816ef06..8fb5e2f617d 100644 --- a/conan/tools/cmake/cmakegen.py +++ b/conan/tools/cmake/cmakegen.py @@ -2,7 +2,7 @@ from conan.tools.env import VirtualEnv -class CMakeGen: +class CMakeGen(object): # Needed for Py2 def __init__(self, conanfile): self.toolchain = CMakeToolchain(conanfile) self.deps = CMakeDeps(conanfile) diff --git a/conans/test/functional/toolchains/env/test_complete.py b/conans/test/functional/toolchains/env/test_complete.py index 986a8b53130..adadefd91de 100644 --- a/conans/test/functional/toolchains/env/test_complete.py +++ b/conans/test/functional/toolchains/env/test_complete.py @@ -33,7 +33,6 @@ def package(self): self.copy("mycmake*", src=src, dst="bin") def package_info(self): - self.cpp_info.exes = ["mycmake"] self.cpp_info.bindirs = ["bin"] """) mycmake_cmakelists = textwrap.dedent(""" diff --git a/conans/test/integration/command/new_test.py b/conans/test/integration/command/new_test.py index 0e3d6e53128..20f38a6061c 100644 --- a/conans/test/integration/command/new_test.py +++ b/conans/test/integration/command/new_test.py @@ -362,6 +362,6 @@ def test_new_v2_cmake(self): conanfile = client.load("conanfile.py") self.assertIn("CMakeToolchain", conanfile) conanfile = client.load("test_package/conanfile.py") - self.assertIn("CMakeToolchain", conanfile) + self.assertIn("CMakeGen", conanfile) cmake = client.load("test_package/CMakeLists.txt") self.assertIn("find_package", cmake) From ac00b00c0fd925e12d5cec0c301f0a04cbab2103 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 18 Mar 2021 14:34:31 +0100 Subject: [PATCH 24/28] remove blanks --- conans/client/graph/graph_builder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py index e37395282ad..026b12998a5 100644 --- a/conans/client/graph/graph_builder.py +++ b/conans/client/graph/graph_builder.py @@ -205,6 +205,7 @@ def _expand_require(self, require, node, graph, check_updates, update, remotes, remotes, profile_host, profile_build, graph_lock, context_switch=context_switch, populate_settings_target=populate_settings_target) + # The closure of a new node starts with just itself new_node.public_closure.add(new_node) new_node.transitive_closure[new_node.name] = new_node From 5247a7425d6fbdd10f26f86a5cfed5e8a6a02b20 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 22 Mar 2021 01:28:11 +0100 Subject: [PATCH 25/28] better visiting with more test --- conan/tools/env/virtualenv.py | 78 ++++++++----------- conans/client/graph/conanfile_dependencies.py | 28 ++----- conans/model/conanfile_interface.py | 4 + .../test/integration/environment/test_env.py | 48 ++++++++++++ 4 files changed, 92 insertions(+), 66 deletions(-) diff --git a/conan/tools/env/virtualenv.py b/conan/tools/env/virtualenv.py index fe6e86c0991..b85b2988ff4 100644 --- a/conan/tools/env/virtualenv.py +++ b/conan/tools/env/virtualenv.py @@ -1,7 +1,7 @@ import platform from conan.tools.env import Environment -from conans.client.graph.graph import CONTEXT_BUILD +from conans.client.graph.graph import CONTEXT_HOST class VirtualEnv: @@ -17,75 +17,61 @@ def build_environment(self): """ collects the buildtime information from dependencies. This is the typical use case of build_requires defining information for consumers """ - # Visit all dependencies - deps_env = Environment() - # TODO: The visitor of dependencies needs to be implemented correctly - # TODO: The environment should probably be composed with direct dependencies first - # TODO: in paths, but this is the opposite - for dep in self._conanfile.dependencies.all: - # environ_info is always "build" - dep_env = dep.buildenv_info - if dep_env is not None: - deps_env.compose(dep_env) - if dep.context == CONTEXT_BUILD: - runenv = self._runenv_from_cpp_info(dep) - deps_env.compose(runenv) + build_env = Environment() + for build_require in self._conanfile.dependencies.build_requires: + for require in build_require.dependencies.requires: + build_env.compose(self._collect_transitive_runenv(require)) + if build_require.buildenv_info: + build_env.compose(build_require.buildenv_info) # The profile environment has precedence, applied last profile_env = self._conanfile.buildenv - deps_env.compose(profile_env) - return deps_env - - def autorun_environment(self): - """ automatically collects the runtime environment from 'cpp_info' from dependencies - By default is enabled and will be captured in 'runenv.xxx' but maybe can be disabled - by parameter or [conf] - """ - dyn_runenv = Environment() - for dep in self._conanfile.dependencies.all: - if dep.context == CONTEXT_BUILD: # Build environment cannot happen in runtime - continue - env = self._runenv_from_cpp_info(dep) - dyn_runenv.compose(env) - return dyn_runenv + build_env.compose(profile_env) + return build_env @staticmethod - def _runenv_from_cpp_info(conanfile_dep): + def _runenv_from_cpp_info(cpp_info): """ return an Environment deducing the runtime information from a cpp_info """ dyn_runenv = Environment() - cpp_info = conanfile_dep.cpp_info if cpp_info is None: # This happens when the dependency is a private one = BINARY_SKIP return dyn_runenv if cpp_info.bin_paths: # cpp_info.exes is not defined yet dyn_runenv.prepend_path("PATH", cpp_info.bin_paths) # If it is a build_require this will be the build-os, otherwise it will be the host-os - os_ = conanfile_dep.settings.get_safe("os") if cpp_info.lib_paths: - if os_ == "Linux": - dyn_runenv.prepend_path("LD_LIBRARY_PATH", cpp_info.lib_paths) - elif os_ == "Macos": - dyn_runenv.prepend_path("DYLD_LIBRARY_PATH", cpp_info.lib_paths) + dyn_runenv.prepend_path("LD_LIBRARY_PATH", cpp_info.lib_paths) + dyn_runenv.prepend_path("DYLD_LIBRARY_PATH", cpp_info.lib_paths) if cpp_info.framework_paths: dyn_runenv.prepend_path("DYLD_FRAMEWORK_PATH", cpp_info.framework_paths) return dyn_runenv + def _collect_transitive_runenv(self, d): + r = Environment() + for child in d.dependencies.requires: + r.compose(self._collect_transitive_runenv(child)) + # Apply "d" runenv, first the implicit + r.compose(self._runenv_from_cpp_info(d.cpp_info)) + # Then the explicit + if d.runenv_info: + r.compose(d.runenv_info) + return r + def run_environment(self): """ collects the runtime information from dependencies. For normal libraries should be very occasional """ - # Visit all dependencies - deps_env = Environment() - for dep in self._conanfile.dependencies.all: - # run_environ_info is always "host" - dep_env = dep.runenv_info - if dep_env is not None: - deps_env.compose(dep_env) + runenv = Environment() + # At the moment we are adding "test-requires" (build_requires in host context) + # to the "runenv", but this will be investigated + for build_require in self._conanfile.dependencies.build_requires: + if build_require.context == CONTEXT_HOST: + runenv.compose(self._collect_transitive_runenv(build_require)) + for require in self._conanfile.dependencies.requires: + runenv.compose(self._collect_transitive_runenv(require)) - autorun = self.autorun_environment() - deps_env.compose(autorun) # FIXME: Missing profile info - result = deps_env + result = runenv return result def generate(self): diff --git a/conans/client/graph/conanfile_dependencies.py b/conans/client/graph/conanfile_dependencies.py index 118bf49a340..1446979bddd 100644 --- a/conans/client/graph/conanfile_dependencies.py +++ b/conans/client/graph/conanfile_dependencies.py @@ -1,5 +1,3 @@ -from collections import OrderedDict - from conans.model.conanfile_interface import ConanFileInterface @@ -7,24 +5,14 @@ class ConanFileDependencies: def __init__(self, node): self._node = node - self._get_ordered_breadth_first_cache = None # TODO: Replace functools Conan 2.0 - def _get_ordered_breadth_first(self): - """ dummy example visitor that returns ordered breadth-first all the deps - """ - if self._get_ordered_breadth_first_cache is None: - result = OrderedDict() # TODO: this is a trick to get an ordered set - open_nodes = self._node.neighbors() - while open_nodes: - new_open = OrderedDict() - for n in open_nodes: - for d in n.neighbors(): - new_open[d] = None - result[n] = None - open_nodes = [n for n in new_open if n not in result] - self._get_ordered_breadth_first_cache = result - return self._get_ordered_breadth_first_cache + @property + def build_requires(self): + return [ConanFileInterface(edge.dst.conanfile) for edge in self._node.dependencies + if edge.build_require] @property - def all(self): - return [ConanFileInterface(n.conanfile) for n in self._get_ordered_breadth_first()] + def requires(self): + # public direct requires + return [ConanFileInterface(edge.dst.conanfile) for edge in self._node.dependencies + if not edge.build_require and not edge.private] diff --git a/conans/model/conanfile_interface.py b/conans/model/conanfile_interface.py index d4e6bd5c11a..45b9c465166 100644 --- a/conans/model/conanfile_interface.py +++ b/conans/model/conanfile_interface.py @@ -28,3 +28,7 @@ def settings(self): @property def context(self): return self._conanfile.context + + @property + def dependencies(self): + return self._conanfile.dependencies diff --git a/conans/test/integration/environment/test_env.py b/conans/test/integration/environment/test_env.py index 97b113e8200..95898b6744b 100644 --- a/conans/test/integration/environment/test_env.py +++ b/conans/test/integration/environment/test_env.py @@ -5,6 +5,7 @@ import pytest from conan.tools.env.environment import environment_wrap_command +from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.tools import TestClient @@ -173,3 +174,50 @@ def generate(self): client.run_command(cmd) assert "MYCOMPILER2!!" in client.out assert "MYPATH2=" in client.out + + +def test_transitive_order(): + gcc = textwrap.dedent(r""" + from conans import ConanFile + class Pkg(ConanFile): + def package_info(self): + self.runenv_info.append("MYVAR", "MyGCCValue") + """) + openssl = textwrap.dedent(r""" + from conans import ConanFile + class Pkg(ConanFile): + settings = "os" + build_requires = "gcc/1.0" + def package_info(self): + self.runenv_info.append("MYVAR", "MyOpenSSL{}Value".format(self.settings.os)) + """) + cmake = textwrap.dedent(r""" + from conans import ConanFile + class Pkg(ConanFile): + requires = "openssl/1.0" + build_requires = "gcc/1.0" + def package_info(self): + self.runenv_info.append("MYVAR", "MyCMakeRunValue") + self.buildenv_info.append("MYVAR", "MyCMakeBuildValue") + """) + client = TestClient() + client.save({"gcc/conanfile.py": gcc, + "cmake/conanfile.py": cmake, + "openssl/conanfile.py": openssl}) + + client.run("create gcc gcc/1.0@") + client.run("create openssl openssl/1.0@ -s os=Windows") + client.run("create openssl openssl/1.0@ -s os=Linux") + client.run("create cmake cmake/1.0@") + + client.save({"conanfile.py": GenConanfile().with_requires("openssl/1.0") + .with_build_requires("cmake/1.0")}, clean_first=True) + client.run("install . -s:b os=Windows -s:h os=Linux -g VirtualEnv") + ext = "bat" if platform.system() == "Windows" else "sh" + buildenv = client.load("conanbuildenv.{}".format(ext)) + assert "MyOpenSSLWindowsValue MyCMakeBuildValue" in buildenv + assert "MyGCCValue" not in buildenv + runenv = client.load("conanrunenv.{}".format(ext)) + assert "MyOpenSSLLinuxValue" in runenv + assert "MyCMake" not in runenv + assert "MyGCCValue" not in runenv From d1be0f653fcc1029f8190666f9208bcd7347ed80 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 22 Mar 2021 01:57:48 +0100 Subject: [PATCH 26/28] fix test --- .../generators/xcode_gcc_vs_test.py | 53 ++++++++++--------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/conans/test/integration/generators/xcode_gcc_vs_test.py b/conans/test/integration/generators/xcode_gcc_vs_test.py index 29c6026a1f4..c988578d52d 100644 --- a/conans/test/integration/generators/xcode_gcc_vs_test.py +++ b/conans/test/integration/generators/xcode_gcc_vs_test.py @@ -1,5 +1,6 @@ import os import re +import textwrap import unittest from conans.model.graph_info import GRAPH_INFO_FILE @@ -15,35 +16,37 @@ class VSXCodeGeneratorsTest(unittest.TestCase): def test_generators(self): ref = ConanFileReference.loads("Hello/0.1@lasote/stable") client = TestClient() - client.save({"conanfile.py": """from conans import ConanFile -import os -class Pkg(ConanFile): - def package(self): - os.makedirs(os.path.join(self.package_folder, "lib")) - os.makedirs(os.path.join(self.package_folder, "include")) - def package_info(self): - self.cpp_info.libs = ["hello"] - self.cpp_info.cxxflags = ["-some_cxx_compiler_flag"] - self.cpp_info.cflags = ["-some_c_compiler_flag"] - self.cpp_info.system_libs = ["system_lib1"] -"""}) + client.save({"conanfile.py": textwrap.dedent(""" + from conans import ConanFile + import os + class Pkg(ConanFile): + def package(self): + os.makedirs(os.path.join(self.package_folder, "lib")) + os.makedirs(os.path.join(self.package_folder, "include")) + def package_info(self): + self.cpp_info.libs = ["hello"] + self.cpp_info.cxxflags = ["-some_cxx_compiler_flag"] + self.cpp_info.cflags = ["-some_c_compiler_flag"] + self.cpp_info.system_libs = ["system_lib1"] + """)}) client.run("export . Hello/0.1@lasote/stable") - conanfile_txt = '''[requires] -Hello/0.1@lasote/stable # My req comment -[generators] -gcc # I need this generator for.. -cmake -visual_studio -xcode -''' + conanfile_txt = textwrap.dedent(''' + [requires] + Hello/0.1@lasote/stable # My req comment + [generators] + gcc # I need this generator for.. + cmake + visual_studio + xcode + ''') client.save({"conanfile.txt": conanfile_txt}, clean_first=True) # Install requirements client.run('install . --build missing') - self.assertEqual(sorted([CONANFILE_TXT, BUILD_INFO_GCC, BUILD_INFO_CMAKE, - BUILD_INFO_VISUAL_STUDIO, BUILD_INFO, - BUILD_INFO_XCODE, CONANINFO, GRAPH_INFO_FILE, LOCKFILE]), - sorted(os.listdir(client.current_folder))) + current_files = os.listdir(client.current_folder) + for f in [CONANFILE_TXT, BUILD_INFO_GCC, BUILD_INFO_CMAKE, BUILD_INFO_VISUAL_STUDIO, + BUILD_INFO, BUILD_INFO_XCODE, CONANINFO, GRAPH_INFO_FILE, LOCKFILE]: + assert f in current_files cmake = client.load(BUILD_INFO_CMAKE) gcc = client.load(BUILD_INFO_GCC) @@ -67,7 +70,7 @@ def package_info(self): from xml.dom import minidom xmldoc = minidom.parse(os.path.join(client.current_folder, BUILD_INFO_VISUAL_STUDIO)) definition_group = xmldoc.getElementsByTagName('ItemDefinitionGroup')[0] - compiler = definition_group.getElementsByTagName("ClCompile")[0] + _ = definition_group.getElementsByTagName("ClCompile")[0] linker = definition_group.getElementsByTagName("Link")[0] def element_content(node): From dca53ca6c41f875b5717f513d05daba401577ede Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 22 Mar 2021 09:38:29 +0100 Subject: [PATCH 27/28] fix test and new test for buildenv from requires --- conan/tools/env/virtualenv.py | 13 ++++ .../test/integration/environment/test_env.py | 78 +++++++++++++++---- 2 files changed, 74 insertions(+), 17 deletions(-) diff --git a/conan/tools/env/virtualenv.py b/conan/tools/env/virtualenv.py index b85b2988ff4..82b78b87263 100644 --- a/conan/tools/env/virtualenv.py +++ b/conan/tools/env/virtualenv.py @@ -18,12 +18,25 @@ def build_environment(self): of build_requires defining information for consumers """ build_env = Environment() + # First visit the direct build_requires for build_require in self._conanfile.dependencies.build_requires: for require in build_require.dependencies.requires: build_env.compose(self._collect_transitive_runenv(require)) if build_require.buildenv_info: build_env.compose(build_require.buildenv_info) + # Requires in host context can also bring some direct buildenv_info + def _collect_transitive_buildenv(d): + r = Environment() + for child in d.dependencies.requires: + r.compose(_collect_transitive_buildenv(child)) + # Then the explicit self + if d.buildenv_info: + r.compose(d.buildenv_info) + return r + for require in self._conanfile.dependencies.requires: + build_env.compose(_collect_transitive_buildenv(require)) + # The profile environment has precedence, applied last profile_env = self._conanfile.buildenv build_env.compose(profile_env) diff --git a/conans/test/integration/environment/test_env.py b/conans/test/integration/environment/test_env.py index 95898b6744b..e1580863a36 100644 --- a/conans/test/integration/environment/test_env.py +++ b/conans/test/integration/environment/test_env.py @@ -5,7 +5,6 @@ import pytest from conan.tools.env.environment import environment_wrap_command -from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.tools import TestClient @@ -205,19 +204,64 @@ def package_info(self): "cmake/conanfile.py": cmake, "openssl/conanfile.py": openssl}) - client.run("create gcc gcc/1.0@") - client.run("create openssl openssl/1.0@ -s os=Windows") - client.run("create openssl openssl/1.0@ -s os=Linux") - client.run("create cmake cmake/1.0@") - - client.save({"conanfile.py": GenConanfile().with_requires("openssl/1.0") - .with_build_requires("cmake/1.0")}, clean_first=True) - client.run("install . -s:b os=Windows -s:h os=Linux -g VirtualEnv") - ext = "bat" if platform.system() == "Windows" else "sh" - buildenv = client.load("conanbuildenv.{}".format(ext)) - assert "MyOpenSSLWindowsValue MyCMakeBuildValue" in buildenv - assert "MyGCCValue" not in buildenv - runenv = client.load("conanrunenv.{}".format(ext)) - assert "MyOpenSSLLinuxValue" in runenv - assert "MyCMake" not in runenv - assert "MyGCCValue" not in runenv + client.run("export gcc gcc/1.0@") + client.run("export openssl openssl/1.0@") + client.run("export cmake cmake/1.0@") + + consumer = textwrap.dedent(r""" + from conans import ConanFile + from conan.tools.env import VirtualEnv + class Pkg(ConanFile): + requires = "openssl/1.0" + build_requires = "cmake/1.0", "gcc/1.0" + def generate(self): + env = VirtualEnv(self) + buildenv = env.build_environment() + self.output.info("BUILDENV: {}!!!".format(buildenv.value("MYVAR"))) + runenv = env.run_environment() + self.output.info("RUNENV: {}!!!".format(runenv.value("MYVAR"))) + """) + client.save({"conanfile.py": consumer}, clean_first=True) + client.run("install . -s:b os=Windows -s:h os=Linux --build -g VirtualEnv") + assert "BUILDENV: MYVAR MyOpenSSLWindowsValue MyCMakeBuildValue!!!" in client.out + assert "RUNENV: MYVAR MyOpenSSLLinuxValue!!!" in client.out + + +def test_buildenv_from_requires(): + openssl = textwrap.dedent(r""" + from conans import ConanFile + class Pkg(ConanFile): + settings = "os" + def package_info(self): + self.buildenv_info.append("OpenSSL_ROOT", "MyOpenSSL{}Value".format(self.settings.os)) + """) + poco = textwrap.dedent(r""" + from conans import ConanFile + class Pkg(ConanFile): + requires = "openssl/1.0" + settings = "os" + def package_info(self): + self.buildenv_info.append("Poco_ROOT", "MyPoco{}Value".format(self.settings.os)) + """) + client = TestClient() + client.save({"poco/conanfile.py": poco, + "openssl/conanfile.py": openssl}) + + client.run("export openssl openssl/1.0@") + client.run("export poco poco/1.0@") + + consumer = textwrap.dedent(r""" + from conans import ConanFile + from conan.tools.env import VirtualEnv + class Pkg(ConanFile): + requires = "poco/1.0" + def generate(self): + env = VirtualEnv(self) + buildenv = env.build_environment() + self.output.info("BUILDENV POCO: {}!!!".format(buildenv.value("Poco_ROOT"))) + self.output.info("BUILDENV OpenSSL: {}!!!".format(buildenv.value("OpenSSL_ROOT"))) + """) + client.save({"conanfile.py": consumer}, clean_first=True) + client.run("install . -s:b os=Windows -s:h os=Linux --build -g VirtualEnv") + assert "BUILDENV POCO: Poco_ROOT MyPocoLinuxValue!!!" in client.out + assert "BUILDENV OpenSSL: OpenSSL_ROOT MyOpenSSLLinuxValue!!!" in client.out From 41bd88907cd2b9340163b1ccd076186507239839 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 23 Mar 2021 21:10:55 +0100 Subject: [PATCH 28/28] making VirtualEnv automatic opt-in --- conan/tools/env/virtualenv.py | 4 ++++ conans/client/generators/__init__.py | 3 ++- conans/test/integration/environment/test_env.py | 6 +++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/conan/tools/env/virtualenv.py b/conan/tools/env/virtualenv.py index 82b78b87263..2c345de163c 100644 --- a/conan/tools/env/virtualenv.py +++ b/conan/tools/env/virtualenv.py @@ -20,8 +20,12 @@ def build_environment(self): build_env = Environment() # First visit the direct build_requires for build_require in self._conanfile.dependencies.build_requires: + # Lower priority, the runenv of all transitive "requires" of the build requires for require in build_require.dependencies.requires: build_env.compose(self._collect_transitive_runenv(require)) + # Second, the implicit self information in build_require.cpp_info + build_env.compose(self._runenv_from_cpp_info(build_require.cpp_info)) + # Finally, higher priority, explicit buildenv_info if build_require.buildenv_info: build_env.compose(build_require.buildenv_info) diff --git a/conans/client/generators/__init__.py b/conans/client/generators/__init__.py index 680f187a9fb..bb5eb3990e9 100644 --- a/conans/client/generators/__init__.py +++ b/conans/client/generators/__init__.py @@ -200,7 +200,8 @@ def write_toolchain(conanfile, path, output): with conanfile_exception_formatter(str(conanfile), "generate"): conanfile.generate() - if conanfile.virtualenv: + # tools.env.virtualenv:auto_use will be always True in Conan 2.0 + if conanfile.conf["tools.env.virtualenv"].auto_use and conanfile.virtualenv: with chdir(path): from conan.tools.env.virtualenv import VirtualEnv env = VirtualEnv(conanfile) diff --git a/conans/test/integration/environment/test_env.py b/conans/test/integration/environment/test_env.py index e1580863a36..5ae6c7bf1cb 100644 --- a/conans/test/integration/environment/test_env.py +++ b/conans/test/integration/environment/test_env.py @@ -6,6 +6,7 @@ from conan.tools.env.environment import environment_wrap_command from conans.test.utils.tools import TestClient +from conans.util.files import save @pytest.fixture() @@ -61,6 +62,7 @@ def package_info(self): self.runenv_info.define("MYGTESTVAR", "MyGTestValue{}".format(self.settings.os)) """) client = TestClient() + save(client.cache.new_config_path, "tools.env.virtualenv:auto_use=True") client.save({"cmake/conanfile.py": cmake, "gtest/conanfile.py": gtest, "openssl/conanfile.py": openssl}) @@ -123,6 +125,7 @@ def build(self): def test_profile_buildenv(): client = TestClient() + save(client.cache.new_config_path, "tools.env.virtualenv:auto_use=True") conanfile = textwrap.dedent("""\ import os, platform from conans import ConanFile @@ -233,7 +236,8 @@ def test_buildenv_from_requires(): class Pkg(ConanFile): settings = "os" def package_info(self): - self.buildenv_info.append("OpenSSL_ROOT", "MyOpenSSL{}Value".format(self.settings.os)) + self.buildenv_info.append("OpenSSL_ROOT", + "MyOpenSSL{}Value".format(self.settings.os)) """) poco = textwrap.dedent(r""" from conans import ConanFile