Create new conan.tools.files (#8550)

* move load

* move save

* move mkdir

* move ftp_downloads

* add rest

* add requester to conanfile

* add test

* test download

* change name

* fix returns

* add test

* back to old tools

* remove test

* add config to conanfile

* separate tools

* simplify function

* minor changes

* minor changes

* absolute imports

* use genconanfile

* revision

* add TODOs

* remove arg

* reemove arg

* Update conans/test/assets/genconanfile.py

Co-authored-by: James <james@conan.io>

* fix tests

Co-authored-by: James <james@conan.io>

conan/tools/files/__init__.py

@@ -0,0 +1 @@
+from conan.tools.files.files import load, save, mkdir, ftp_download, download, get

conan/tools/files/files.py

@@ -0,0 +1,152 @@
+import errno
+import os
+
+from conans.errors import ConanException
+from conans.util.files import decode_text, to_file_bytes
+from conans.client.tools.files import unzip
+from conans.client.downloaders.download import run_downloader
+
+
+def load(conanfile, path, binary=False, encoding="auto"):
+    """ Loads a file content """
+    with open(path, 'rb') as handle:
+        tmp = handle.read()
+        # TODO: Get rid of encoding auto-detection
+        return tmp if binary else decode_text(tmp, encoding)
+
+
+def save(conanfile, path, content, append=False):
+    if append:
+        mode = "ab"
+        try:
+            os.makedirs(os.path.dirname(path))
+        except Exception:
+            pass
+    else:
+        mode = "wb"
+        dir_path = os.path.dirname(path)
+        if not os.path.isdir(dir_path):
+            try:
+                os.makedirs(dir_path)
+            except OSError as error:
+                if error.errno not in (errno.EEXIST, errno.ENOENT):
+                    raise OSError("The folder {} does not exist and could not be created ({})."
+                                  .format(dir_path, error.strerror))
+            except Exception:
+                raise
+
+    with open(path, mode) as handle:
+        handle.write(to_file_bytes(content, encoding="utf-8"))
+
+
+def mkdir(conanfile, path):
+    """Recursive mkdir, doesnt fail if already existing"""
+    if os.path.exists(path):
+        return
+    os.makedirs(path)
+
+
+def get(conanfile, url, md5='', sha1='', sha256='', destination=".", filename="", keep_permissions=False,
+        pattern=None, verify=True, retry=None, retry_wait=None,
+        overwrite=False, auth=None, headers=None, strip_root=False):
+    """ high level downloader + unzipper + (optional hash checker) + delete temporary zip
+    """
+    requester = conanfile._conan_requester
+    output = conanfile.output
+    if not filename:  # deduce filename from the URL
+        url_base = url[0] if isinstance(url, (list, tuple)) else url
+        if "?" in url_base or "=" in url_base:
+            raise ConanException("Cannot deduce file name from the url: '{}'. Use 'filename' "
+                                 "parameter.".format(url_base))
+        filename = os.path.basename(url_base)
+
+    download(url, filename, out=output, requester=requester, verify=verify,
+             retry=retry, retry_wait=retry_wait, overwrite=overwrite, auth=auth, headers=headers,
+             md5=md5, sha1=sha1, sha256=sha256)
+    unzip(filename, destination=destination, keep_permissions=keep_permissions, pattern=pattern,
+          output=output, strip_root=strip_root)
+    os.unlink(filename)
+
+
+def ftp_download(conanfile, ip, filename, login='', password=''):
+    import ftplib
+    try:
+        ftp = ftplib.FTP(ip)
+        ftp.login(login, password)
+        filepath, filename = os.path.split(filename)
+        if filepath:
+            ftp.cwd(filepath)
+        with open(filename, 'wb') as f:
+            ftp.retrbinary('RETR ' + filename, f.write)
+    except Exception as e:
+        try:
+            os.unlink(filename)
+        except OSError:
+            pass
+        raise ConanException("Error in FTP download from %s\n%s" % (ip, str(e)))
+    finally:
+        try:
+            ftp.quit()
+        except Exception:
+            pass
+
+
+def download(conanfile, url, filename, verify=True, out=None, retry=None, retry_wait=None, overwrite=False,
+             auth=None, headers=None, requester=None, md5='', sha1='', sha256=''):
+    """Retrieves a file from a given URL into a file with a given filename.
+       It uses certificates from a list of known verifiers for https downloads,
+       but this can be optionally disabled.
+
+    :param url: URL to download. It can be a list, which only the first one will be downloaded, and
+                the follow URLs will be used as mirror in case of download error.
+    :param filename: Name of the file to be created in the local storage
+    :param verify: When False, disables https certificate validation
+    :param out: An object with a write() method can be passed to get the output. stdout will use if
+                not specified
+    :param retry: Number of retries in case of failure. Default is overriden by general.retry in the
+                  conan.conf file or an env variable CONAN_RETRY
+    :param retry_wait: Seconds to wait between download attempts. Default is overriden by
+                       general.retry_wait in the conan.conf file or an env variable CONAN_RETRY_WAIT
+    :param overwrite: When True, Conan will overwrite the destination file if exists. Otherwise it
+                      will raise an exception
+    :param auth: A tuple of user and password to use HTTPBasic authentication
+    :param headers: A dictionary with additional headers
+    :param requester: HTTP requests instance
+    :param md5: MD5 hash code to check the downloaded file
+    :param sha1: SHA-1 hash code to check the downloaded file
+    :param sha256: SHA-256 hash code to check the downloaded file
+    :return: None
+    """
+    # TODO: Add all parameters to the new conf
+    out = conanfile.output
+    requester = conanfile._conan_requester
+    config = conanfile.conf
+
+    # It might be possible that users provide their own requester
+    retry = retry if retry is not None else int(config["tools.files.download"].retry)
+    retry = retry if retry is not None else 1
+    retry_wait = retry_wait if retry_wait is not None else int(config["tools.files.download"].retry_wait)
+    retry_wait = retry_wait if retry_wait is not None else 5
+
+    checksum = sha256 or sha1 or md5
+
+    def _download_file(file_url):
+        # The download cache is only used if a checksum is provided, otherwise, a normal download
+        run_downloader(requester=requester, output=out, verify=verify, config=config,
+                       user_download=True, use_cache=bool(config and checksum), url=file_url,
+                       file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite,
+                       auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256)
+        out.writeln("")
+
+    if not isinstance(url, (list, tuple)):
+        _download_file(url)
+    else:  # We were provided several URLs to try
+        for url_it in url:
+            try:
+                _download_file(url_it)
+                break
+            except Exception as error:
+                message = "Could not download from the URL {}: {}.".format(url_it, str(error))
+                out.warn(message + " Trying another mirror.")
+        else:
+            raise ConanException("All downloads from ({}) URLs have failed.".format(len(url)))

conans/client/conan_api.py

@@ -203,7 +203,7 @@ def __init__(self, cache_folder, user_io, http_requester=None, runner=None, quie
                                                   self.generator_manager)
         self.pyreq_loader = PyRequireLoader(self.proxy, self.range_resolver)
         self.loader = ConanFileLoader(self.runner, self.out, self.python_requires,
-                                      self.generator_manager, self.pyreq_loader)
+                                      self.generator_manager, self.pyreq_loader, self.requester)
 
         self.binaries_analyzer = GraphBinariesAnalyzer(self.cache, self.out, self.remote_manager)
         self.graph_manager = GraphManager(self.out, self.cache, self.remote_manager, self.loader,

conans/client/loader.py

@@ -23,14 +23,16 @@
 
 class ConanFileLoader(object):
 
-    def __init__(self, runner, output, python_requires, generator_manager=None, pyreq_loader=None):
+    def __init__(self, runner, output, python_requires, generator_manager=None, pyreq_loader=None,
+                 requester=None):
         self._runner = runner
         self._generator_manager = generator_manager
         self._output = output
         self._pyreq_loader = pyreq_loader
         self._python_requires = python_requires
         sys.modules["conans"].python_requires = python_requires
         self._cached_conanfile_classes = {}
+        self._requester = requester
 
     def load_basic(self, conanfile_path, lock_python_requires=None, user=None, channel=None,
                    display=""):
@@ -45,7 +47,8 @@ def load_basic_module(self, conanfile_path, lock_python_requires=None, user=None
         """
         cached = self._cached_conanfile_classes.get(conanfile_path)
         if cached and cached[1] == lock_python_requires:
-            conanfile = cached[0](self._output, self._runner, display, user, channel)
+            conanfile = cached[0](self._output, self._runner, display, user, channel,
+                                  self._requester)
             if hasattr(conanfile, "init") and callable(conanfile.init):
                 with conanfile_exception_formatter(str(conanfile), "init"):
                     conanfile.init()
@@ -82,7 +85,7 @@ def load_basic_module(self, conanfile_path, lock_python_requires=None, user=None
 
             self._cached_conanfile_classes[conanfile_path] = (conanfile, lock_python_requires,
                                                               module)
-            result = conanfile(self._output, self._runner, display, user, channel)
+            result = conanfile(self._output, self._runner, display, user, channel, self._requester)
             if hasattr(result, "init") and callable(result.init):
                 with conanfile_exception_formatter(str(result), "init"):
                     result.init()

conans/model/conan_file.py

@@ -133,7 +133,7 @@ class ConanFile(object):
     # layout
     layout = None
 
-    def __init__(self, output, runner, display_name="", user=None, channel=None):
+    def __init__(self, output, runner, display_name="", user=None, channel=None, requester=None):
         # an output stream (writeln, info, warn error)
         self.output = ScopedOutput(display_name, output)
         self.display_name = display_name
@@ -144,6 +144,7 @@ def __init__(self, output, runner, display_name="", user=None, channel=None):
 
         self.compatible_packages = []
         self._conan_using_build_profile = False
+        self._conan_requester = requester
 
         self.layout = Layout()
 

conans/test/assets/genconanfile.py

@@ -410,6 +410,6 @@ def __repr__(self):
             ret.append("    {}".format(self._package_id_method))
         if self._test_method:
             ret.append("    {}".format(self._test_method))
-        if len(ret) == 2:
+        if ret[-1] == "class HelloConan(ConanFile):":
             ret.append("    pass")
         return "\n".join(ret)

conans/test/functional/tools/test_files.py

@@ -0,0 +1,71 @@
+import os
+import textwrap
+
+from bottle import static_file
+
+from conans.test.utils.test_files import temp_folder
+from conans.test.utils.tools import TestClient, StoppableThreadBottle
+from conans.util.files import save
+from conans.test.assets.genconanfile import GenConanfile
+
+
+class TestConanToolFiles:
+
+    def test_imports(self):
+        conanfile = GenConanfile().with_import("from conan.tools.files import load, save, "
+                                               "mkdir, download, get, ftp_download")
+        client = TestClient()
+        client.save({"conanfile.py": conanfile})
+        client.run("install .")
+
+    def test_load_save_mkdir(self):
+        conanfile = textwrap.dedent("""
+            from conans import ConanFile
+            from conan.tools.files import load, save, mkdir
+
+            class Pkg(ConanFile):
+                name = "mypkg"
+                version = "1.0"
+                def source(self):
+                    mkdir(self, "myfolder")
+                    save(self, "./myfolder/myfile", "some_content")
+                    assert load(self, "./myfolder/myfile") == "some_content"
+            """)
+        client = TestClient()
+        client.save({"conanfile.py": conanfile})
+        client.run("source .")
+
+    def test_download(self):
+        http_server = StoppableThreadBottle()
+        file_path = os.path.join(temp_folder(), "myfile.txt")
+        save(file_path, "some content")
+
+        @http_server.server.get("/myfile.txt")
+        def get_file():
+            return static_file(os.path.basename(file_path), os.path.dirname(file_path))
+
+        http_server.run_server()
+
+        profile = textwrap.dedent("""\
+            [conf]
+            tools.files.download:retry=1
+            tools.files.download:retry_wait=0
+            """)
+
+        conanfile = textwrap.dedent("""
+            import os
+            from conans import ConanFile
+            from conan.tools.files import download
+
+            class Pkg(ConanFile):
+                name = "mypkg"
+                version = "1.0"
+                def source(self):
+                    download(self, "http://localhost:{}/myfile.txt", "myfile.txt")
+                    assert os.path.exists("myfile.txt")
+            """.format(http_server.port))
+
+        client = TestClient()
+        client.save({"conanfile.py": conanfile})
+        client.save({"profile": profile})
+        client.run("create . -pr=profile")