From e7fe5f0a1c096f091ee4806b4d6b1f6ffd0ebdc0 Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Mon, 1 Mar 2021 10:14:33 +0100 Subject: [PATCH] Create new conan.tools.files (#8550) * move load * move save * move mkdir * move ftp_downloads * add rest * add requester to conanfile * add test * test download * change name * fix returns * add test * back to old tools * remove test * add config to conanfile * separate tools * simplify function * minor changes * minor changes * absolute imports * use genconanfile * revision * add TODOs * remove arg * reemove arg * Update conans/test/assets/genconanfile.py Co-authored-by: James * fix tests Co-authored-by: James --- conan/tools/files/__init__.py | 1 + conan/tools/files/files.py | 152 ++++++++++++++++++ conans/client/conan_api.py | 2 +- conans/client/loader.py | 9 +- conans/model/conan_file.py | 3 +- conans/test/assets/genconanfile.py | 2 +- conans/test/functional/tools/test_files.py | 71 ++++++++ .../integration/layout/test_local_commands.py | 87 +++++----- 8 files changed, 281 insertions(+), 46 deletions(-) create mode 100644 conan/tools/files/__init__.py create mode 100644 conan/tools/files/files.py create mode 100644 conans/test/functional/tools/test_files.py diff --git a/conan/tools/files/__init__.py b/conan/tools/files/__init__.py new file mode 100644 index 00000000000..007aaef3fb4 --- /dev/null +++ b/conan/tools/files/__init__.py @@ -0,0 +1 @@ +from conan.tools.files.files import load, save, mkdir, ftp_download, download, get diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py new file mode 100644 index 00000000000..86a3855aefb --- /dev/null +++ b/conan/tools/files/files.py @@ -0,0 +1,152 @@ +import errno +import os + +from conans.errors import ConanException +from conans.util.files import decode_text, to_file_bytes +from conans.client.tools.files import unzip +from conans.client.downloaders.download import run_downloader + + +def load(conanfile, path, binary=False, encoding="auto"): + """ Loads a file content """ + with open(path, 'rb') as handle: + tmp = handle.read() + # TODO: Get rid of encoding auto-detection + return tmp if binary else decode_text(tmp, encoding) + + +def save(conanfile, path, content, append=False): + if append: + mode = "ab" + try: + os.makedirs(os.path.dirname(path)) + except Exception: + pass + else: + mode = "wb" + dir_path = os.path.dirname(path) + if not os.path.isdir(dir_path): + try: + os.makedirs(dir_path) + except OSError as error: + if error.errno not in (errno.EEXIST, errno.ENOENT): + raise OSError("The folder {} does not exist and could not be created ({})." + .format(dir_path, error.strerror)) + except Exception: + raise + + with open(path, mode) as handle: + handle.write(to_file_bytes(content, encoding="utf-8")) + + +def mkdir(conanfile, path): + """Recursive mkdir, doesnt fail if already existing""" + if os.path.exists(path): + return + os.makedirs(path) + + +def get(conanfile, url, md5='', sha1='', sha256='', destination=".", filename="", keep_permissions=False, + pattern=None, verify=True, retry=None, retry_wait=None, + overwrite=False, auth=None, headers=None, strip_root=False): + """ high level downloader + unzipper + (optional hash checker) + delete temporary zip + """ + requester = conanfile._conan_requester + output = conanfile.output + if not filename: # deduce filename from the URL + url_base = url[0] if isinstance(url, (list, tuple)) else url + if "?" in url_base or "=" in url_base: + raise ConanException("Cannot deduce file name from the url: '{}'. Use 'filename' " + "parameter.".format(url_base)) + filename = os.path.basename(url_base) + + download(url, filename, out=output, requester=requester, verify=verify, + retry=retry, retry_wait=retry_wait, overwrite=overwrite, auth=auth, headers=headers, + md5=md5, sha1=sha1, sha256=sha256) + unzip(filename, destination=destination, keep_permissions=keep_permissions, pattern=pattern, + output=output, strip_root=strip_root) + os.unlink(filename) + + +def ftp_download(conanfile, ip, filename, login='', password=''): + import ftplib + try: + ftp = ftplib.FTP(ip) + ftp.login(login, password) + filepath, filename = os.path.split(filename) + if filepath: + ftp.cwd(filepath) + with open(filename, 'wb') as f: + ftp.retrbinary('RETR ' + filename, f.write) + except Exception as e: + try: + os.unlink(filename) + except OSError: + pass + raise ConanException("Error in FTP download from %s\n%s" % (ip, str(e))) + finally: + try: + ftp.quit() + except Exception: + pass + + +def download(conanfile, url, filename, verify=True, out=None, retry=None, retry_wait=None, overwrite=False, + auth=None, headers=None, requester=None, md5='', sha1='', sha256=''): + """Retrieves a file from a given URL into a file with a given filename. + It uses certificates from a list of known verifiers for https downloads, + but this can be optionally disabled. + + :param url: URL to download. It can be a list, which only the first one will be downloaded, and + the follow URLs will be used as mirror in case of download error. + :param filename: Name of the file to be created in the local storage + :param verify: When False, disables https certificate validation + :param out: An object with a write() method can be passed to get the output. stdout will use if + not specified + :param retry: Number of retries in case of failure. Default is overriden by general.retry in the + conan.conf file or an env variable CONAN_RETRY + :param retry_wait: Seconds to wait between download attempts. Default is overriden by + general.retry_wait in the conan.conf file or an env variable CONAN_RETRY_WAIT + :param overwrite: When True, Conan will overwrite the destination file if exists. Otherwise it + will raise an exception + :param auth: A tuple of user and password to use HTTPBasic authentication + :param headers: A dictionary with additional headers + :param requester: HTTP requests instance + :param md5: MD5 hash code to check the downloaded file + :param sha1: SHA-1 hash code to check the downloaded file + :param sha256: SHA-256 hash code to check the downloaded file + :return: None + """ + # TODO: Add all parameters to the new conf + out = conanfile.output + requester = conanfile._conan_requester + config = conanfile.conf + + # It might be possible that users provide their own requester + retry = retry if retry is not None else int(config["tools.files.download"].retry) + retry = retry if retry is not None else 1 + retry_wait = retry_wait if retry_wait is not None else int(config["tools.files.download"].retry_wait) + retry_wait = retry_wait if retry_wait is not None else 5 + + checksum = sha256 or sha1 or md5 + + def _download_file(file_url): + # The download cache is only used if a checksum is provided, otherwise, a normal download + run_downloader(requester=requester, output=out, verify=verify, config=config, + user_download=True, use_cache=bool(config and checksum), url=file_url, + file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, + auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) + out.writeln("") + + if not isinstance(url, (list, tuple)): + _download_file(url) + else: # We were provided several URLs to try + for url_it in url: + try: + _download_file(url_it) + break + except Exception as error: + message = "Could not download from the URL {}: {}.".format(url_it, str(error)) + out.warn(message + " Trying another mirror.") + else: + raise ConanException("All downloads from ({}) URLs have failed.".format(len(url))) diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index de48a9a8f99..95b0456360c 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -203,7 +203,7 @@ def __init__(self, cache_folder, user_io, http_requester=None, runner=None, quie self.generator_manager) self.pyreq_loader = PyRequireLoader(self.proxy, self.range_resolver) self.loader = ConanFileLoader(self.runner, self.out, self.python_requires, - self.generator_manager, self.pyreq_loader) + self.generator_manager, self.pyreq_loader, self.requester) self.binaries_analyzer = GraphBinariesAnalyzer(self.cache, self.out, self.remote_manager) self.graph_manager = GraphManager(self.out, self.cache, self.remote_manager, self.loader, diff --git a/conans/client/loader.py b/conans/client/loader.py index 9851fb6e42e..78e1a6fee01 100644 --- a/conans/client/loader.py +++ b/conans/client/loader.py @@ -23,7 +23,8 @@ class ConanFileLoader(object): - def __init__(self, runner, output, python_requires, generator_manager=None, pyreq_loader=None): + def __init__(self, runner, output, python_requires, generator_manager=None, pyreq_loader=None, + requester=None): self._runner = runner self._generator_manager = generator_manager self._output = output @@ -31,6 +32,7 @@ def __init__(self, runner, output, python_requires, generator_manager=None, pyre self._python_requires = python_requires sys.modules["conans"].python_requires = python_requires self._cached_conanfile_classes = {} + self._requester = requester def load_basic(self, conanfile_path, lock_python_requires=None, user=None, channel=None, display=""): @@ -45,7 +47,8 @@ def load_basic_module(self, conanfile_path, lock_python_requires=None, user=None """ cached = self._cached_conanfile_classes.get(conanfile_path) if cached and cached[1] == lock_python_requires: - conanfile = cached[0](self._output, self._runner, display, user, channel) + conanfile = cached[0](self._output, self._runner, display, user, channel, + self._requester) if hasattr(conanfile, "init") and callable(conanfile.init): with conanfile_exception_formatter(str(conanfile), "init"): conanfile.init() @@ -82,7 +85,7 @@ def load_basic_module(self, conanfile_path, lock_python_requires=None, user=None self._cached_conanfile_classes[conanfile_path] = (conanfile, lock_python_requires, module) - result = conanfile(self._output, self._runner, display, user, channel) + result = conanfile(self._output, self._runner, display, user, channel, self._requester) if hasattr(result, "init") and callable(result.init): with conanfile_exception_formatter(str(result), "init"): result.init() diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index 077c33de2de..a8720376a03 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -133,7 +133,7 @@ class ConanFile(object): # layout layout = None - def __init__(self, output, runner, display_name="", user=None, channel=None): + def __init__(self, output, runner, display_name="", user=None, channel=None, requester=None): # an output stream (writeln, info, warn error) self.output = ScopedOutput(display_name, output) self.display_name = display_name @@ -144,6 +144,7 @@ def __init__(self, output, runner, display_name="", user=None, channel=None): self.compatible_packages = [] self._conan_using_build_profile = False + self._conan_requester = requester self.layout = Layout() diff --git a/conans/test/assets/genconanfile.py b/conans/test/assets/genconanfile.py index 217cdbee026..33a2bc20b1c 100644 --- a/conans/test/assets/genconanfile.py +++ b/conans/test/assets/genconanfile.py @@ -410,6 +410,6 @@ def __repr__(self): ret.append(" {}".format(self._package_id_method)) if self._test_method: ret.append(" {}".format(self._test_method)) - if len(ret) == 2: + if ret[-1] == "class HelloConan(ConanFile):": ret.append(" pass") return "\n".join(ret) diff --git a/conans/test/functional/tools/test_files.py b/conans/test/functional/tools/test_files.py new file mode 100644 index 00000000000..a680dd5a066 --- /dev/null +++ b/conans/test/functional/tools/test_files.py @@ -0,0 +1,71 @@ +import os +import textwrap + +from bottle import static_file + +from conans.test.utils.test_files import temp_folder +from conans.test.utils.tools import TestClient, StoppableThreadBottle +from conans.util.files import save +from conans.test.assets.genconanfile import GenConanfile + + +class TestConanToolFiles: + + def test_imports(self): + conanfile = GenConanfile().with_import("from conan.tools.files import load, save, " + "mkdir, download, get, ftp_download") + client = TestClient() + client.save({"conanfile.py": conanfile}) + client.run("install .") + + def test_load_save_mkdir(self): + conanfile = textwrap.dedent(""" + from conans import ConanFile + from conan.tools.files import load, save, mkdir + + class Pkg(ConanFile): + name = "mypkg" + version = "1.0" + def source(self): + mkdir(self, "myfolder") + save(self, "./myfolder/myfile", "some_content") + assert load(self, "./myfolder/myfile") == "some_content" + """) + client = TestClient() + client.save({"conanfile.py": conanfile}) + client.run("source .") + + def test_download(self): + http_server = StoppableThreadBottle() + file_path = os.path.join(temp_folder(), "myfile.txt") + save(file_path, "some content") + + @http_server.server.get("/myfile.txt") + def get_file(): + return static_file(os.path.basename(file_path), os.path.dirname(file_path)) + + http_server.run_server() + + profile = textwrap.dedent("""\ + [conf] + tools.files.download:retry=1 + tools.files.download:retry_wait=0 + """) + + conanfile = textwrap.dedent(""" + import os + from conans import ConanFile + from conan.tools.files import download + + class Pkg(ConanFile): + name = "mypkg" + version = "1.0" + def source(self): + download(self, "http://localhost:{}/myfile.txt", "myfile.txt") + assert os.path.exists("myfile.txt") + """.format(http_server.port)) + + client = TestClient() + client.save({"conanfile.py": conanfile}) + client.save({"profile": profile}) + client.run("create . -pr=profile") diff --git a/conans/test/integration/layout/test_local_commands.py b/conans/test/integration/layout/test_local_commands.py index 375ef6c44a7..747b0b88cad 100644 --- a/conans/test/integration/layout/test_local_commands.py +++ b/conans/test/integration/layout/test_local_commands.py @@ -1,4 +1,5 @@ import os +import textwrap from conans.model.ref import ConanFileReference, PackageReference from conans.test.assets.genconanfile import GenConanfile @@ -101,28 +102,31 @@ def test_export_pkg(): # FIXME: The configure is not valid to change the layout, we need the settings and options # ready client = TestClient() - conan_file = str(GenConanfile().with_import("from conans import tools")) - conan_file += """ - no_copy_source = True + conan_file = textwrap.dedent(""" + from conans import ConanFile + from conans import tools + + class HelloConan(ConanFile): + no_copy_source = True - def configure(self): - self.layout.source.folder = "my_source" - self.layout.build.folder = "my_build" + def configure(self): + self.layout.source.folder = "my_source" + self.layout.build.folder = "my_build" - def source(self): - tools.save("downloaded.h", "bar") + def source(self): + tools.save("downloaded.h", "bar") - def build(self): - tools.save("library.lib", "bar") - tools.save("generated.h", "bar") + def build(self): + tools.save("library.lib", "bar") + tools.save("generated.h", "bar") - def package(self): - self.output.warn("Source folder: {}".format(self.source_folder)) - self.output.warn("Build folder: {}".format(self.build_folder)) - self.output.warn("Package folder: {}".format(self.package_folder)) - self.copy("*.h") - self.copy("*.lib") - """ + def package(self): + self.output.warn("Source folder: {}".format(self.source_folder)) + self.output.warn("Build folder: {}".format(self.build_folder)) + self.output.warn("Package folder: {}".format(self.package_folder)) + self.copy("*.h") + self.copy("*.lib") + """) client.save({"conanfile.py": conan_file}) client.run("install . -if=my_install") @@ -149,28 +153,31 @@ def test_export_pkg_local(): # FIXME: The configure is not valid to change the layout, we need the settings and options # ready client = TestClient() - conan_file = str(GenConanfile().with_import("from conans import tools")) - conan_file += """ - no_copy_source = True - - def configure(self): - self.layout.source.folder = "my_source" - self.layout.build.folder = "my_build" - - def source(self): - tools.save("downloaded.h", "bar") - - def build(self): - tools.save("library.lib", "bar") - tools.save("generated.h", "bar") - - def package(self): - self.output.warn("Source folder: {}".format(self.source_folder)) - self.output.warn("Build folder: {}".format(self.build_folder)) - self.output.warn("Package folder: {}".format(self.package_folder)) - self.copy("*.h") - self.copy("*.lib") - """ + conan_file = textwrap.dedent(""" + from conans import ConanFile + from conans import tools + + class HelloConan(ConanFile): + no_copy_source = True + + def configure(self): + self.layout.source.folder = "my_source" + self.layout.build.folder = "my_build" + + def source(self): + tools.save("downloaded.h", "bar") + + def build(self): + tools.save("library.lib", "bar") + tools.save("generated.h", "bar") + + def package(self): + self.output.warn("Source folder: {}".format(self.source_folder)) + self.output.warn("Build folder: {}".format(self.build_folder)) + self.output.warn("Package folder: {}".format(self.package_folder)) + self.copy("*.h") + self.copy("*.lib") + """) client.save({"conanfile.py": conan_file}) client.run("install . -if=my_install")