From ca75b0765ebe42005b93d71180be01e273c0bfbf Mon Sep 17 00:00:00 2001 From: memsharded Date: Fri, 6 Nov 2020 17:32:17 +0100 Subject: [PATCH 1/4] added new lock install command --- conans/client/command.py | 9 +++++- conans/client/conan_api.py | 29 +++++++++++++++++++ .../graph_lock/lock_install_test.py | 26 +++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 conans/test/functional/graph_lock/lock_install_test.py diff --git a/conans/client/command.py b/conans/client/command.py index 659cd68a6c3..cc7cea0a0cf 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -1878,6 +1878,11 @@ def lock(self, *args): clean_modified_cmd = subparsers.add_parser('clean-modified', help='Clean modified flags') clean_modified_cmd.add_argument('lockfile', help='Path to the lockfile') + install_cmd = subparsers.add_parser('install', help='Install a lockfile') + install_cmd.add_argument('lockfile', help='Path to the lockfile') + install_cmd.add_argument("-g", "--generator", nargs=1, action=Extender, + help='Generators to use') + create_cmd = subparsers.add_parser('create', help='Create a lockfile from a conanfile or a reference') create_cmd.add_argument("path", nargs="?", help="Path to a conanfile") @@ -1903,7 +1908,9 @@ def lock(self, *args): args = parser.parse_args(*args) self._warn_python_version() - if args.subcommand == "update": + if args.subcommand == "install": + self._conan.lock_install(args.lockfile, generators=args.generator) + elif args.subcommand == "update": self._conan.lock_update(args.old_lockfile, args.new_lockfile) elif args.subcommand == "build-order": build_order = self._conan.lock_build_order(args.lockfile) diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index 798fbe3ef35..7217e76c941 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -1316,6 +1316,35 @@ def lock_clean_modified(self, lockfile, cwd=None): graph_lock.clean_modified() graph_lock_file.save(lockfile) + @api_method + def lock_install(self, lockfile, remote_name=None, build=None, + generators=None, install_folder=None, cwd=None, + lockfile_out=None): + lockfile = _make_abs_path(lockfile, cwd) if lockfile else None + graph_info = get_graph_info(None, None, cwd, None, + self.app.cache, self.app.out, lockfile=lockfile) + + if not generators: # We don't want the default txt + generators = False + + install_folder = _make_abs_path(install_folder, cwd) + + mkdir(install_folder) + remotes = self.app.load_remotes(remote_name=remote_name) + recorder = ActionRecorder() + graph_lock = graph_info.graph_lock + root_id = graph_lock.root_node_id() + reference = graph_lock.nodes[root_id].ref + deps_install(self.app, ref_or_path=reference, install_folder=install_folder, + remotes=remotes, graph_info=graph_info, build_modes=build, + generators=generators, recorder=recorder) + + if lockfile_out: + lockfile_out = _make_abs_path(lockfile_out, cwd) + graph_lock_file = GraphLockFile(graph_info.profile_host, graph_info.profile_build, + graph_info.graph_lock) + graph_lock_file.save(lockfile_out) + @api_method def lock_create(self, path, lockfile_out, reference=None, name=None, version=None, user=None, channel=None, diff --git a/conans/test/functional/graph_lock/lock_install_test.py b/conans/test/functional/graph_lock/lock_install_test.py new file mode 100644 index 00000000000..7a4b26cada3 --- /dev/null +++ b/conans/test/functional/graph_lock/lock_install_test.py @@ -0,0 +1,26 @@ +import os +import unittest + +from conans.test.utils.tools import TestClient, GenConanfile + + +class LockInstallTest(unittest.TestCase): + + def test_install(self): + client = TestClient() + client.save({"conanfile.py": GenConanfile("PkgA", "0.1").with_package_file("file.h", "0.1")}) + client.run("create . user/channel") + + # Use a consumer with a version range + client.save({"conanfile.py": + GenConanfile("PkgB", "0.1").with_require("PkgA/[>=0.1]@user/channel")}) + client.run("create . user/channel") + client.run("lock create --reference=PkgB/0.1@user/channel --lockfile-out=lock1.lock") + + client.save({"conanfile.py": GenConanfile("PkgA", "0.2").with_package_file("file.h", "0.2")}) + client.run("create . user/channel") + + client.run("lock install lock1.lock -g deploy") + self.assertIn("PkgA/0.1@user/channel from local cache", client.out) + file_h = client.load("PkgA/file.h") + self.assertEqual(file_h, "0.1") From 2361127abb410eecbea06385dac0d05a67202adb Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 16 Mar 2021 15:59:55 +0100 Subject: [PATCH 2/4] merged --- conans/test/functional/graph_lock/lock_install_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conans/test/functional/graph_lock/lock_install_test.py b/conans/test/functional/graph_lock/lock_install_test.py index ac9b155387e..13de28f3edf 100644 --- a/conans/test/functional/graph_lock/lock_install_test.py +++ b/conans/test/functional/graph_lock/lock_install_test.py @@ -1,4 +1,3 @@ -import os import unittest from conans.test.utils.tools import TestClient, GenConanfile From 61dd9673432a76e34e7126f8f6bec1baf30715c9 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 17 Mar 2021 22:35:25 +0100 Subject: [PATCH 3/4] working --- conans/client/command.py | 4 +- conans/client/conan_api.py | 16 +++-- conans/client/manager.py | 53 +++++++++++----- .../graph_lock/lock_install_test.py | 61 +++++++++++++------ 4 files changed, 95 insertions(+), 39 deletions(-) diff --git a/conans/client/command.py b/conans/client/command.py index 362fe650269..b0c37493c66 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -1886,6 +1886,8 @@ def lock(self, *args): install_cmd = subparsers.add_parser('install', help='Install a lockfile') install_cmd.add_argument('lockfile', help='Path to the lockfile') + install_cmd.add_argument("--recipes", action="store_true", + help="Install only recipes, not binaries") install_cmd.add_argument("-g", "--generator", nargs=1, action=Extender, help='Generators to use') @@ -1932,7 +1934,7 @@ def lock(self, *args): self._warn_python_version() if args.subcommand == "install": - self._conan.lock_install(args.lockfile, generators=args.generator) + self._conan.lock_install(args.lockfile, generators=args.generator, recipes=args.recipes) elif args.subcommand == "update": self._conan.lock_update(args.old_lockfile, args.new_lockfile) elif args.subcommand == "bundle": diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index 6877c148dd1..b4ca725beef 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -34,7 +34,7 @@ from conans.client.importer import run_imports, undo_imports from conans.client.installer import BinaryInstaller from conans.client.loader import ConanFileLoader -from conans.client.manager import deps_install +from conans.client.manager import deps_install, compute_dependency_graph from conans.client.migrations import ClientMigrator from conans.client.output import ConanOutput, colorama_initialize from conans.client.profile_loader import profile_from_args, read_profile @@ -1345,7 +1345,7 @@ def lock_clean_modified(self, lockfile, cwd=None): @api_method def lock_install(self, lockfile, remote_name=None, build=None, generators=None, install_folder=None, cwd=None, - lockfile_out=None): + lockfile_out=None, recipes=None): lockfile = _make_abs_path(lockfile, cwd) if lockfile else None graph_info = get_graph_info(None, None, cwd, None, self.app.cache, self.app.out, lockfile=lockfile) @@ -1361,9 +1361,15 @@ def lock_install(self, lockfile, remote_name=None, build=None, graph_lock = graph_info.graph_lock root_id = graph_lock.root_node_id() reference = graph_lock.nodes[root_id].ref - deps_install(self.app, ref_or_path=reference, install_folder=install_folder, - remotes=remotes, graph_info=graph_info, build_modes=build, - generators=generators, recorder=recorder) + if recipes: + graph = compute_dependency_graph(self.app, ref_or_path=reference, remotes=remotes, + graph_info=graph_info, build_modes=build, + generators=generators, recorder=recorder) + print_graph(graph, self.app.out) + else: + deps_install(self.app, ref_or_path=reference, install_folder=install_folder, + remotes=remotes, graph_info=graph_info, build_modes=build, + generators=generators, recorder=recorder) if lockfile_out: lockfile_out = _make_abs_path(lockfile_out, cwd) diff --git a/conans/client/manager.py b/conans/client/manager.py index 7ff82470d53..865e8fc1455 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -23,22 +23,34 @@ def deps_install(app, ref_or_path, install_folder, graph_info, remotes=None, bui manifest_interactive=False, generators=None, no_imports=False, create_reference=None, keep_build=False, recorder=None, lockfile_node_id=None): """ Fetch and build all dependencies for the given reference - @param app: The ConanApp instance with all collaborators - @param ref_or_path: ConanFileReference or path to user space conanfile - @param install_folder: where the output files will be saved - @param build_modes: List of build_modes specified - @param update: Check for updated in the upstream remotes (and update) - @param manifest_folder: Folder to install the manifests - @param manifest_verify: Verify dependencies manifests against stored ones - @param manifest_interactive: Install deps manifests in folder for later verify, asking user - for confirmation - @param generators: List of generators from command line. If False, no generator will be - written - @param no_imports: Install specified packages but avoid running imports - - """ + @param app: The ConanApp instance with all collaborators + @param ref_or_path: ConanFileReference or path to user space conanfile + @param install_folder: where the output files will be saved + @param build_modes: List of build_modes specified + @param update: Check for updated in the upstream remotes (and update) + @param manifest_folder: Folder to install the manifests + @param manifest_verify: Verify dependencies manifests against stored ones + @param manifest_interactive: Install deps manifests in folder for later verify, asking user + for confirmation + @param generators: List of generators from command line. If False, no generator will be + written + @param no_imports: Install specified packages but avoid running imports + + """ + graph = compute_dependency_graph(app, ref_or_path, graph_info, remotes, build_modes, update, + generators, create_reference, recorder, lockfile_node_id) + + install_dependency_graph(app, graph, ref_or_path, install_folder, graph_info, remotes, + build_modes, update, manifest_folder, manifest_verify, + manifest_interactive, generators, no_imports, create_reference, + keep_build, recorder) + + +def compute_dependency_graph(app, ref_or_path, graph_info, remotes=None, build_modes=None, + update=False, generators=None, create_reference=None, recorder=None, + lockfile_node_id=None): + out, user_io, graph_manager, cache = app.out, app.user_io, app.graph_manager, app.cache - remote_manager, hook_manager = app.remote_manager, app.hook_manager profile_host, profile_build = graph_info.profile_host, graph_info.profile_build if generators is not False: @@ -57,6 +69,17 @@ def deps_install(app, ref_or_path, install_folder, graph_info, remotes=None, bui deps_graph = graph_manager.load_graph(ref_or_path, create_reference, graph_info, build_modes, False, update, remotes, recorder, lockfile_node_id=lockfile_node_id) + return deps_graph + + +def install_dependency_graph(app, deps_graph, ref_or_path, install_folder, graph_info, remotes=None, + build_modes=None, update=False, manifest_folder=None, + manifest_verify=False, manifest_interactive=False, generators=None, + no_imports=False, create_reference=None, keep_build=False, + recorder=None): + out, user_io, graph_manager, cache = app.out, app.user_io, app.graph_manager, app.cache + remote_manager, hook_manager = app.remote_manager, app.hook_manager + profile_host, profile_build = graph_info.profile_host, graph_info.profile_build graph_lock = graph_info.graph_lock # After the graph is loaded it is defined root_node = deps_graph.root conanfile = root_node.conanfile diff --git a/conans/test/functional/graph_lock/lock_install_test.py b/conans/test/functional/graph_lock/lock_install_test.py index 13de28f3edf..c654554d21c 100644 --- a/conans/test/functional/graph_lock/lock_install_test.py +++ b/conans/test/functional/graph_lock/lock_install_test.py @@ -1,26 +1,51 @@ -import unittest from conans.test.utils.tools import TestClient, GenConanfile -class LockInstallTest(unittest.TestCase): +def test_install(): + client = TestClient() + client.save({"conanfile.py": GenConanfile("pkga", "0.1").with_package_file("file.h", "0.1")}) + client.run("create . user/channel") - def test_install(self): - client = TestClient() - client.save({"conanfile.py": GenConanfile("pkga", "0.1").with_package_file("file.h", "0.1")}) - client.run("create . user/channel") + # Use a consumer with a version range + client.save({"conanfile.py": + GenConanfile("pkgb", "0.1").with_require("pkga/[>=0.1]@user/channel")}) + client.run("create . user/channel") + client.run("lock create --reference=pkgb/0.1@user/channel --lockfile-out=lock1.lock") - # Use a consumer with a version range - client.save({"conanfile.py": - GenConanfile("pkgb", "0.1").with_require("pkga/[>=0.1]@user/channel")}) - client.run("create . user/channel") - client.run("lock create --reference=pkgb/0.1@user/channel --lockfile-out=lock1.lock") + # We can create a pkga/0.2, but it will not be used + client.save({"conanfile.py": GenConanfile("pkga", "0.2").with_package_file("file.h", "0.2")}) + client.run("create . user/channel") - # We can create a pkga/0.2, but it will not be used - client.save({"conanfile.py": GenConanfile("pkga", "0.2").with_package_file("file.h", "0.2")}) - client.run("create . user/channel") + client.run("lock install lock1.lock -g deploy") + assert "pkga/0.1@user/channel from local cache" in client.out + file_h = client.load("pkga/file.h") + assert file_h == "0.1" + + +def test_install_recipes(): + client = TestClient(default_server_user=True) + client.save({"conanfile.py": GenConanfile("pkga", "0.1").with_package_file("file.h", "0.1")}) + client.run("create . user/channel") + + # Use a consumer with a version range + client.save({"conanfile.py": + GenConanfile("pkgb", "0.1").with_require("pkga/[>=0.1]@user/channel")}) + client.run("create . user/channel") + client.run("lock create --reference=pkgb/0.1@user/channel --lockfile-out=lock1.lock") + + # We can create a pkga/0.2, but it will not be used + client.save({"conanfile.py": GenConanfile("pkga", "0.2").with_package_file("file.h", "0.2")}) + client.run("create . user/channel") + + client.run("lock install lock1.lock --recipes") + assert "pkga/0.1@user/channel:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Cache" in client.out + assert "pkgb/0.1@user/channel:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Cache" in client.out + + client.run("upload * --all -c") + client.run("remove * -f") + client.run("lock install lock1.lock --recipes") + + assert "pkga/0.1@user/channel:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Download" in client.out + assert "pkgb/0.1@user/channel:cfd10f60aeaa00f5ca1f90b5fe97c3fe19e7ec23 - Download" in client.out - client.run("lock install lock1.lock -g deploy") - self.assertIn("pkga/0.1@user/channel from local cache", client.out) - file_h = client.load("pkga/file.h") - self.assertEqual(file_h, "0.1") From 310493e37cc7fd712123bb9398b875edc5f75699 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 17 Mar 2021 22:49:58 +0100 Subject: [PATCH 4/4] lock install --recipes for speed --- conans/client/conan_api.py | 12 +++-- conans/client/manager.py | 51 +++++-------------- .../graph_lock/lock_install_test.py | 1 - 3 files changed, 21 insertions(+), 43 deletions(-) diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index b4ca725beef..bda95283f0e 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -34,7 +34,7 @@ from conans.client.importer import run_imports, undo_imports from conans.client.installer import BinaryInstaller from conans.client.loader import ConanFileLoader -from conans.client.manager import deps_install, compute_dependency_graph +from conans.client.manager import deps_install from conans.client.migrations import ClientMigrator from conans.client.output import ConanOutput, colorama_initialize from conans.client.profile_loader import profile_from_args, read_profile @@ -1362,14 +1362,16 @@ def lock_install(self, lockfile, remote_name=None, build=None, root_id = graph_lock.root_node_id() reference = graph_lock.nodes[root_id].ref if recipes: - graph = compute_dependency_graph(self.app, ref_or_path=reference, remotes=remotes, - graph_info=graph_info, build_modes=build, - generators=generators, recorder=recorder) + graph = self.app.graph_manager.load_graph(reference, create_reference=None, + graph_info=graph_info, build_mode=None, + check_updates=False, update=None, + remotes=remotes, recorder=recorder, + lockfile_node_id=root_id) print_graph(graph, self.app.out) else: deps_install(self.app, ref_or_path=reference, install_folder=install_folder, remotes=remotes, graph_info=graph_info, build_modes=build, - generators=generators, recorder=recorder) + generators=generators, recorder=recorder, lockfile_node_id=root_id) if lockfile_out: lockfile_out = _make_abs_path(lockfile_out, cwd) diff --git a/conans/client/manager.py b/conans/client/manager.py index 865e8fc1455..558eb247665 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -23,34 +23,22 @@ def deps_install(app, ref_or_path, install_folder, graph_info, remotes=None, bui manifest_interactive=False, generators=None, no_imports=False, create_reference=None, keep_build=False, recorder=None, lockfile_node_id=None): """ Fetch and build all dependencies for the given reference - @param app: The ConanApp instance with all collaborators - @param ref_or_path: ConanFileReference or path to user space conanfile - @param install_folder: where the output files will be saved - @param build_modes: List of build_modes specified - @param update: Check for updated in the upstream remotes (and update) - @param manifest_folder: Folder to install the manifests - @param manifest_verify: Verify dependencies manifests against stored ones - @param manifest_interactive: Install deps manifests in folder for later verify, asking user - for confirmation - @param generators: List of generators from command line. If False, no generator will be - written - @param no_imports: Install specified packages but avoid running imports - - """ - graph = compute_dependency_graph(app, ref_or_path, graph_info, remotes, build_modes, update, - generators, create_reference, recorder, lockfile_node_id) - - install_dependency_graph(app, graph, ref_or_path, install_folder, graph_info, remotes, - build_modes, update, manifest_folder, manifest_verify, - manifest_interactive, generators, no_imports, create_reference, - keep_build, recorder) - - -def compute_dependency_graph(app, ref_or_path, graph_info, remotes=None, build_modes=None, - update=False, generators=None, create_reference=None, recorder=None, - lockfile_node_id=None): + @param app: The ConanApp instance with all collaborators + @param ref_or_path: ConanFileReference or path to user space conanfile + @param install_folder: where the output files will be saved + @param build_modes: List of build_modes specified + @param update: Check for updated in the upstream remotes (and update) + @param manifest_folder: Folder to install the manifests + @param manifest_verify: Verify dependencies manifests against stored ones + @param manifest_interactive: Install deps manifests in folder for later verify, asking user + for confirmation + @param generators: List of generators from command line. If False, no generator will be + written + @param no_imports: Install specified packages but avoid running imports + """ out, user_io, graph_manager, cache = app.out, app.user_io, app.graph_manager, app.cache + remote_manager, hook_manager = app.remote_manager, app.hook_manager profile_host, profile_build = graph_info.profile_host, graph_info.profile_build if generators is not False: @@ -69,17 +57,6 @@ def compute_dependency_graph(app, ref_or_path, graph_info, remotes=None, build_m deps_graph = graph_manager.load_graph(ref_or_path, create_reference, graph_info, build_modes, False, update, remotes, recorder, lockfile_node_id=lockfile_node_id) - return deps_graph - - -def install_dependency_graph(app, deps_graph, ref_or_path, install_folder, graph_info, remotes=None, - build_modes=None, update=False, manifest_folder=None, - manifest_verify=False, manifest_interactive=False, generators=None, - no_imports=False, create_reference=None, keep_build=False, - recorder=None): - out, user_io, graph_manager, cache = app.out, app.user_io, app.graph_manager, app.cache - remote_manager, hook_manager = app.remote_manager, app.hook_manager - profile_host, profile_build = graph_info.profile_host, graph_info.profile_build graph_lock = graph_info.graph_lock # After the graph is loaded it is defined root_node = deps_graph.root conanfile = root_node.conanfile diff --git a/conans/test/functional/graph_lock/lock_install_test.py b/conans/test/functional/graph_lock/lock_install_test.py index c654554d21c..b7561387c39 100644 --- a/conans/test/functional/graph_lock/lock_install_test.py +++ b/conans/test/functional/graph_lock/lock_install_test.py @@ -1,4 +1,3 @@ - from conans.test.utils.tools import TestClient, GenConanfile