From 866417b5bd5dbc4ecd9b58f4cbe759362cfc0204 Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Mon, 4 Jul 2022 15:09:01 +0100 Subject: [PATCH 1/8] Add ability to reference files in the local filesystem with file:// specifier --- conan/tools/files/files.py | 19 +++++++++---- .../unittests/tools/files/test_downloads.py | 27 +++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index 7eca6af893f..97b2b7c8c08 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -11,6 +11,7 @@ from fnmatch import fnmatch import six +import urllib from conan.tools import CONAN_TOOLCHAIN_ARGS_FILE, CONAN_TOOLCHAIN_ARGS_SECTION from conan.tools.apple.apple import is_apple_os @@ -164,13 +165,21 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None, download_cache = config["tools.files.download:download_cache"] if checksum else None def _download_file(file_url): - # The download cache is only used if a checksum is provided, otherwise, a normal download - run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache, - user_download=True, url=file_url, - file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, - auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) + if file_url.startswith('file:'): + filepath = _path_from_file_uri(file_url) + shutil.copyfile(filepath, filename) + else: + # The download cache is only used if a checksum is provided, otherwise, a normal download + run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache, + user_download=True, url=file_url, + file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, + auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) out.writeln("") + def _path_from_file_uri(uri): + path = urllib.parse.urlparse(uri).path + return urllib.request.url2pathname(path) + if not isinstance(url, (list, tuple)): _download_file(url) else: # We were provided several URLs to try diff --git a/conans/test/unittests/tools/files/test_downloads.py b/conans/test/unittests/tools/files/test_downloads.py index 30675c976e2..473e92ced47 100644 --- a/conans/test/unittests/tools/files/test_downloads.py +++ b/conans/test/unittests/tools/files/test_downloads.py @@ -1,4 +1,5 @@ import os +import platform import pytest import requests @@ -157,6 +158,32 @@ def test_download_no_retries_errors(self, bottle_server): assert "Waiting" not in str(conanfile.output) assert "retry" not in str(conanfile.output) + def test_download_localfile(self, fs): + conanfile = ConanFileMock() + conanfile._conan_requester = requests + + file_location = '/path/to/file.txt' + if platform.system() == "Windows": + file_location = "C:" + file_location + fs.create_file(file_location, contents='this is some content') + file_url = f"file:///{file_location}" + + dest = os.path.join(temp_folder(), "file.txt") + download(conanfile, file_url, dest) + content = load(dest) + assert "this is some content" == content + + def test_download_localfile_notfound(self): + conanfile = ConanFileMock() + conanfile._conan_requester = requests + + file_url = "file:///path/to/missing/file.txt" + dest = os.path.join(temp_folder(), "file.txt") + + with pytest.raises(FileNotFoundError) as exc: + download(conanfile, file_url, dest) + + assert "No such file" in str(exc.value) @pytest.fixture() def bottle_server_zip(): From a182818ca9fbaa6155c7b3820a2ea35e6b000832 Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Mon, 4 Jul 2022 15:26:10 +0100 Subject: [PATCH 2/8] Add pyfakefs to requirements_dev --- conans/requirements_dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/conans/requirements_dev.txt b/conans/requirements_dev.txt index a40bd8883b0..25b0af0c709 100644 --- a/conans/requirements_dev.txt +++ b/conans/requirements_dev.txt @@ -1,6 +1,7 @@ pytest>=6.1.1, <7.0.0; python_version > '3.0' pytest>=4.6.11; python_version < '3.0' pytest-xdist # To launch in N cores with pytest -n +pyfakefs>=4.5.6 parameterized>=0.6.3 mock>=1.3.0, <1.4.0 WebTest>=2.0.18, <2.1.0 From 130b58860ab1239dc38deddfe7f6c910c3d98ae5 Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 6 Jul 2022 11:21:08 +0100 Subject: [PATCH 3/8] Add implementation for local file downloader --- conan/tools/files/files.py | 19 ++++-------- conans/client/downloaders/download.py | 9 ++++-- .../downloaders/local_file_downloader.py | 29 +++++++++++++++++++ .../unittests/tools/files/test_downloads.py | 7 +++-- 4 files changed, 45 insertions(+), 19 deletions(-) create mode 100644 conans/client/downloaders/local_file_downloader.py diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index 97b2b7c8c08..1e76324377c 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -165,21 +165,14 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None, download_cache = config["tools.files.download:download_cache"] if checksum else None def _download_file(file_url): - if file_url.startswith('file:'): - filepath = _path_from_file_uri(file_url) - shutil.copyfile(filepath, filename) - else: - # The download cache is only used if a checksum is provided, otherwise, a normal download - run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache, - user_download=True, url=file_url, - file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, - auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) + # The download cache is only used if a checksum is provided, otherwise, a normal download + local_filesystem = True if file_url.startswith("file:") else False + run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache, + user_download=True, url=file_url, local_filesystem=local_filesystem, + file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, + auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) out.writeln("") - def _path_from_file_uri(uri): - path = urllib.parse.urlparse(uri).path - return urllib.request.url2pathname(path) - if not isinstance(url, (list, tuple)): _download_file(url) else: # We were provided several URLs to try diff --git a/conans/client/downloaders/download.py b/conans/client/downloaders/download.py index c7ff5c1ecf8..1bc9939fb17 100644 --- a/conans/client/downloaders/download.py +++ b/conans/client/downloaders/download.py @@ -1,11 +1,14 @@ from conans.client.downloaders.cached_file_downloader import CachedFileDownloader from conans.client.downloaders.file_downloader import FileDownloader +from conans.client.downloaders.local_file_downloader import LocalFileDownloader -def run_downloader(requester, output, verify, retry, retry_wait, download_cache, user_download=False, - **kwargs): +def run_downloader(requester, output, verify, retry, retry_wait, download_cache, local_filesystem, + user_download=False, **kwargs): downloader = FileDownloader(requester=requester, output=output, verify=verify, config_retry=retry, config_retry_wait=retry_wait) - if download_cache: + if local_filesystem: + downloader = LocalFileDownloader(output=output) + elif download_cache: downloader = CachedFileDownloader(download_cache, downloader, user_download=user_download) return downloader.download(**kwargs) diff --git a/conans/client/downloaders/local_file_downloader.py b/conans/client/downloaders/local_file_downloader.py new file mode 100644 index 00000000000..233cc5974a5 --- /dev/null +++ b/conans/client/downloaders/local_file_downloader.py @@ -0,0 +1,29 @@ +import os +from urllib.parse import urlparse +from urllib.request import url2pathname +from shutil import copyfile + +from conans.client.tools.files import check_md5, check_sha1, check_sha256 +from conans.errors import ConanException + + +class LocalFileDownloader(object): + + def __init__(self, output): + self._output = output + + def download(self, url, file_path, md5=None, sha1=None, sha256=None, **kwargs): + + file_origin = self._path_from_file_uri(url) + copyfile(file_origin, file_path) + + if md5: + check_md5(file_path, md5) + if sha1: + check_sha1(file_path, sha1) + if sha256: + check_sha256(file_path, sha256) + + def _path_from_file_uri(self, uri): + path = urlparse(uri).path + return url2pathname(path) diff --git a/conans/test/unittests/tools/files/test_downloads.py b/conans/test/unittests/tools/files/test_downloads.py index 473e92ced47..4e746f04858 100644 --- a/conans/test/unittests/tools/files/test_downloads.py +++ b/conans/test/unittests/tools/files/test_downloads.py @@ -165,13 +165,14 @@ def test_download_localfile(self, fs): file_location = '/path/to/file.txt' if platform.system() == "Windows": file_location = "C:" + file_location - fs.create_file(file_location, contents='this is some content') + fs.create_file(file_location, contents=b'this is some content\n') file_url = f"file:///{file_location}" + file_md5 = "a0b156435474e688206c68e5c66a3327" dest = os.path.join(temp_folder(), "file.txt") - download(conanfile, file_url, dest) + download(conanfile, file_url, dest, md5=file_md5) content = load(dest) - assert "this is some content" == content + assert "this is some content" == content.rstrip() def test_download_localfile_notfound(self): conanfile = ConanFileMock() From 9b7f613c991976f323daaebb66f6e5ee989ab572 Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 6 Jul 2022 11:57:02 +0100 Subject: [PATCH 4/8] Move implementation to files.py --- conan/tools/files/files.py | 30 +++++++++++++++---- conans/client/downloaders/download.py | 5 +--- .../downloaders/local_file_downloader.py | 29 ------------------ 3 files changed, 25 insertions(+), 39 deletions(-) delete mode 100644 conans/client/downloaders/local_file_downloader.py diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index 1e76324377c..90c4803bf6d 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -11,7 +11,8 @@ from fnmatch import fnmatch import six -import urllib +from urllib.parse import urlparse +from urllib.request import url2pathname from conan.tools import CONAN_TOOLCHAIN_ARGS_FILE, CONAN_TOOLCHAIN_ARGS_SECTION from conan.tools.apple.apple import is_apple_os @@ -166,11 +167,13 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None, def _download_file(file_url): # The download cache is only used if a checksum is provided, otherwise, a normal download - local_filesystem = True if file_url.startswith("file:") else False - run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache, - user_download=True, url=file_url, local_filesystem=local_filesystem, - file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, - auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) + if file_url.startswith("file:"): + _copy_local_file(url=file_url, file_path=filename, md5=md5, sha1=sha1, sha256=sha256) + else: + run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache, + user_download=True, url=file_url, + file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, + auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) out.writeln("") if not isinstance(url, (list, tuple)): @@ -186,6 +189,21 @@ def _download_file(file_url): else: raise ConanException("All downloads from ({}) URLs have failed.".format(len(url))) + def _copy_local_file(url, file_path, md5=None, sha1=None, sha256=None): + file_origin = _path_from_file_uri(url) + shutil.copyfile(file_origin, file_path) + + if md5: + check_md5(file_path, md5) + if sha1: + check_sha1(file_path, sha1) + if sha256: + check_sha256(file_path, sha256) + + def _path_from_file_uri(self, uri): + path = urlparse(uri).path + return url2pathname(path) + def rename(conanfile, src, dst): """ diff --git a/conans/client/downloaders/download.py b/conans/client/downloaders/download.py index 1bc9939fb17..9458c6e35f1 100644 --- a/conans/client/downloaders/download.py +++ b/conans/client/downloaders/download.py @@ -1,14 +1,11 @@ from conans.client.downloaders.cached_file_downloader import CachedFileDownloader from conans.client.downloaders.file_downloader import FileDownloader -from conans.client.downloaders.local_file_downloader import LocalFileDownloader def run_downloader(requester, output, verify, retry, retry_wait, download_cache, local_filesystem, user_download=False, **kwargs): downloader = FileDownloader(requester=requester, output=output, verify=verify, config_retry=retry, config_retry_wait=retry_wait) - if local_filesystem: - downloader = LocalFileDownloader(output=output) - elif download_cache: + if download_cache: downloader = CachedFileDownloader(download_cache, downloader, user_download=user_download) return downloader.download(**kwargs) diff --git a/conans/client/downloaders/local_file_downloader.py b/conans/client/downloaders/local_file_downloader.py deleted file mode 100644 index 233cc5974a5..00000000000 --- a/conans/client/downloaders/local_file_downloader.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -from urllib.parse import urlparse -from urllib.request import url2pathname -from shutil import copyfile - -from conans.client.tools.files import check_md5, check_sha1, check_sha256 -from conans.errors import ConanException - - -class LocalFileDownloader(object): - - def __init__(self, output): - self._output = output - - def download(self, url, file_path, md5=None, sha1=None, sha256=None, **kwargs): - - file_origin = self._path_from_file_uri(url) - copyfile(file_origin, file_path) - - if md5: - check_md5(file_path, md5) - if sha1: - check_sha1(file_path, sha1) - if sha256: - check_sha256(file_path, sha256) - - def _path_from_file_uri(self, uri): - path = urlparse(uri).path - return url2pathname(path) From cf2429436a0fb0f3b0f7ebeec2b278766d4706c9 Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 6 Jul 2022 12:04:15 +0100 Subject: [PATCH 5/8] Fix implementation --- conan/tools/files/files.py | 26 +++++++++++++------------- conans/client/downloaders/download.py | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index 90c4803bf6d..3f69573b979 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -168,7 +168,7 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None, def _download_file(file_url): # The download cache is only used if a checksum is provided, otherwise, a normal download if file_url.startswith("file:"): - _copy_local_file(url=file_url, file_path=filename, md5=md5, sha1=sha1, sha256=sha256) + _copy_local_file_from_uri(conanfile, url=file_url, file_path=filename, md5=md5, sha1=sha1, sha256=sha256) else: run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache, user_download=True, url=file_url, @@ -189,20 +189,20 @@ def _download_file(file_url): else: raise ConanException("All downloads from ({}) URLs have failed.".format(len(url))) - def _copy_local_file(url, file_path, md5=None, sha1=None, sha256=None): - file_origin = _path_from_file_uri(url) - shutil.copyfile(file_origin, file_path) +def _copy_local_file_from_uri(conanfile, url, file_path, md5=None, sha1=None, sha256=None): + file_origin = _path_from_file_uri(url) + shutil.copyfile(file_origin, file_path) - if md5: - check_md5(file_path, md5) - if sha1: - check_sha1(file_path, sha1) - if sha256: - check_sha256(file_path, sha256) + if md5: + check_md5(conanfile, file_path, md5) + if sha1: + check_sha1(conanfile, file_path, sha1) + if sha256: + check_sha256(conanfile, file_path, sha256) - def _path_from_file_uri(self, uri): - path = urlparse(uri).path - return url2pathname(path) +def _path_from_file_uri(uri): + path = urlparse(uri).path + return url2pathname(path) def rename(conanfile, src, dst): diff --git a/conans/client/downloaders/download.py b/conans/client/downloaders/download.py index 9458c6e35f1..a477aa91e91 100644 --- a/conans/client/downloaders/download.py +++ b/conans/client/downloaders/download.py @@ -2,7 +2,7 @@ from conans.client.downloaders.file_downloader import FileDownloader -def run_downloader(requester, output, verify, retry, retry_wait, download_cache, local_filesystem, +def run_downloader(requester, output, verify, retry, retry_wait, download_cache, user_download=False, **kwargs): downloader = FileDownloader(requester=requester, output=output, verify=verify, config_retry=retry, config_retry_wait=retry_wait) From db618c9b5c2f0379ddf52294d36904366895d44d Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 6 Jul 2022 13:16:15 +0100 Subject: [PATCH 6/8] Use temp folder instead of pyfakefs --- conans/requirements_dev.txt | 1 - conans/test/unittests/tools/files/test_downloads.py | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/conans/requirements_dev.txt b/conans/requirements_dev.txt index 25b0af0c709..a40bd8883b0 100644 --- a/conans/requirements_dev.txt +++ b/conans/requirements_dev.txt @@ -1,7 +1,6 @@ pytest>=6.1.1, <7.0.0; python_version > '3.0' pytest>=4.6.11; python_version < '3.0' pytest-xdist # To launch in N cores with pytest -n -pyfakefs>=4.5.6 parameterized>=0.6.3 mock>=1.3.0, <1.4.0 WebTest>=2.0.18, <2.1.0 diff --git a/conans/test/unittests/tools/files/test_downloads.py b/conans/test/unittests/tools/files/test_downloads.py index 4e746f04858..29ae7447cab 100644 --- a/conans/test/unittests/tools/files/test_downloads.py +++ b/conans/test/unittests/tools/files/test_downloads.py @@ -158,18 +158,18 @@ def test_download_no_retries_errors(self, bottle_server): assert "Waiting" not in str(conanfile.output) assert "retry" not in str(conanfile.output) - def test_download_localfile(self, fs): + def test_download_localfile(self): conanfile = ConanFileMock() conanfile._conan_requester = requests - file_location = '/path/to/file.txt' - if platform.system() == "Windows": - file_location = "C:" + file_location - fs.create_file(file_location, contents=b'this is some content\n') + file_location = os.path.join(temp_folder(), "file.txt") + with open(file_location, 'w') as textfile: + textfile.write('this is some content\n') + file_url = f"file:///{file_location}" file_md5 = "a0b156435474e688206c68e5c66a3327" - dest = os.path.join(temp_folder(), "file.txt") + dest = os.path.join(temp_folder(), "downloaded_file.txt") download(conanfile, file_url, dest, md5=file_md5) content = load(dest) assert "this is some content" == content.rstrip() From c3f820ecaef24a52e010d8e4a8a4c1d901b55806 Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 6 Jul 2022 13:17:26 +0100 Subject: [PATCH 7/8] Undo redundant changes in download.py --- conans/client/downloaders/download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conans/client/downloaders/download.py b/conans/client/downloaders/download.py index a477aa91e91..c7ff5c1ecf8 100644 --- a/conans/client/downloaders/download.py +++ b/conans/client/downloaders/download.py @@ -2,8 +2,8 @@ from conans.client.downloaders.file_downloader import FileDownloader -def run_downloader(requester, output, verify, retry, retry_wait, download_cache, - user_download=False, **kwargs): +def run_downloader(requester, output, verify, retry, retry_wait, download_cache, user_download=False, + **kwargs): downloader = FileDownloader(requester=requester, output=output, verify=verify, config_retry=retry, config_retry_wait=retry_wait) if download_cache: From 4e8508baeddc4a631031dace80862bfd87e48c08 Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 6 Jul 2022 16:37:05 +0100 Subject: [PATCH 8/8] Fix file loading and md5 checksum --- conans/test/unittests/tools/files/test_downloads.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/conans/test/unittests/tools/files/test_downloads.py b/conans/test/unittests/tools/files/test_downloads.py index 29ae7447cab..1ec8077ca78 100644 --- a/conans/test/unittests/tools/files/test_downloads.py +++ b/conans/test/unittests/tools/files/test_downloads.py @@ -163,16 +163,15 @@ def test_download_localfile(self): conanfile._conan_requester = requests file_location = os.path.join(temp_folder(), "file.txt") - with open(file_location, 'w') as textfile: - textfile.write('this is some content\n') + save(file_location, "this is some content") file_url = f"file:///{file_location}" - file_md5 = "a0b156435474e688206c68e5c66a3327" + file_md5 = "736db904ad222bf88ee6b8d103fceb8e" dest = os.path.join(temp_folder(), "downloaded_file.txt") download(conanfile, file_url, dest, md5=file_md5) content = load(dest) - assert "this is some content" == content.rstrip() + assert "this is some content" == content def test_download_localfile_notfound(self): conanfile = ConanFileMock()