From b20b72beb77c063d8c27c55db37425349e232a26 Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Thu, 7 Jul 2022 11:42:09 +0100 Subject: [PATCH] =?UTF-8?q?Add=20ability=20to=20reference=20files=20in=20t?= =?UTF-8?q?he=20local=20filesystem=20with=20file://=20s=E2=80=A6=20(#11569?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add ability to reference files in the local filesystem with file:// specifier * Add pyfakefs to requirements_dev * Add implementation for local file downloader * Move implementation to files.py * Fix implementation * Use temp folder instead of pyfakefs * Undo redundant changes in download.py * Fix file loading and md5 checksum --- conan/tools/files/files.py | 28 ++++++++++++++++--- .../unittests/tools/files/test_downloads.py | 27 ++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index 7eca6af893f..3f69573b979 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -11,6 +11,8 @@ from fnmatch import fnmatch import six +from urllib.parse import urlparse +from urllib.request import url2pathname from conan.tools import CONAN_TOOLCHAIN_ARGS_FILE, CONAN_TOOLCHAIN_ARGS_SECTION from conan.tools.apple.apple import is_apple_os @@ -165,10 +167,13 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None, def _download_file(file_url): # The download cache is only used if a checksum is provided, otherwise, a normal download - run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache, - user_download=True, url=file_url, - file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, - auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) + if file_url.startswith("file:"): + _copy_local_file_from_uri(conanfile, url=file_url, file_path=filename, md5=md5, sha1=sha1, sha256=sha256) + else: + run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache, + user_download=True, url=file_url, + file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, + auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) out.writeln("") if not isinstance(url, (list, tuple)): @@ -184,6 +189,21 @@ def _download_file(file_url): else: raise ConanException("All downloads from ({}) URLs have failed.".format(len(url))) +def _copy_local_file_from_uri(conanfile, url, file_path, md5=None, sha1=None, sha256=None): + file_origin = _path_from_file_uri(url) + shutil.copyfile(file_origin, file_path) + + if md5: + check_md5(conanfile, file_path, md5) + if sha1: + check_sha1(conanfile, file_path, sha1) + if sha256: + check_sha256(conanfile, file_path, sha256) + +def _path_from_file_uri(uri): + path = urlparse(uri).path + return url2pathname(path) + def rename(conanfile, src, dst): """ diff --git a/conans/test/unittests/tools/files/test_downloads.py b/conans/test/unittests/tools/files/test_downloads.py index 30675c976e2..1ec8077ca78 100644 --- a/conans/test/unittests/tools/files/test_downloads.py +++ b/conans/test/unittests/tools/files/test_downloads.py @@ -1,4 +1,5 @@ import os +import platform import pytest import requests @@ -157,6 +158,32 @@ def test_download_no_retries_errors(self, bottle_server): assert "Waiting" not in str(conanfile.output) assert "retry" not in str(conanfile.output) + def test_download_localfile(self): + conanfile = ConanFileMock() + conanfile._conan_requester = requests + + file_location = os.path.join(temp_folder(), "file.txt") + save(file_location, "this is some content") + + file_url = f"file:///{file_location}" + file_md5 = "736db904ad222bf88ee6b8d103fceb8e" + + dest = os.path.join(temp_folder(), "downloaded_file.txt") + download(conanfile, file_url, dest, md5=file_md5) + content = load(dest) + assert "this is some content" == content + + def test_download_localfile_notfound(self): + conanfile = ConanFileMock() + conanfile._conan_requester = requests + + file_url = "file:///path/to/missing/file.txt" + dest = os.path.join(temp_folder(), "file.txt") + + with pytest.raises(FileNotFoundError) as exc: + download(conanfile, file_url, dest) + + assert "No such file" in str(exc.value) @pytest.fixture() def bottle_server_zip():