Skip to content

Commit

Permalink
Add ability to reference files in the local filesystem with file:// s… (
Browse files Browse the repository at this point in the history
#11569)

* Add ability to reference files in the local filesystem with file:// specifier

* Add pyfakefs to requirements_dev

* Add implementation for local file downloader

* Move implementation to files.py

* Fix implementation

* Use temp folder instead of pyfakefs

* Undo redundant changes in download.py

* Fix file loading and md5 checksum
  • Loading branch information
jcar87 committed Jul 7, 2022
1 parent 3123a9a commit b20b72b
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 4 deletions.
28 changes: 24 additions & 4 deletions conan/tools/files/files.py
Expand Up @@ -11,6 +11,8 @@
from fnmatch import fnmatch

import six
from urllib.parse import urlparse
from urllib.request import url2pathname

from conan.tools import CONAN_TOOLCHAIN_ARGS_FILE, CONAN_TOOLCHAIN_ARGS_SECTION
from conan.tools.apple.apple import is_apple_os
Expand Down Expand Up @@ -165,10 +167,13 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None,

def _download_file(file_url):
# The download cache is only used if a checksum is provided, otherwise, a normal download
run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache,
user_download=True, url=file_url,
file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite,
auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256)
if file_url.startswith("file:"):
_copy_local_file_from_uri(conanfile, url=file_url, file_path=filename, md5=md5, sha1=sha1, sha256=sha256)
else:
run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache,
user_download=True, url=file_url,
file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite,
auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256)
out.writeln("")

if not isinstance(url, (list, tuple)):
Expand All @@ -184,6 +189,21 @@ def _download_file(file_url):
else:
raise ConanException("All downloads from ({}) URLs have failed.".format(len(url)))

def _copy_local_file_from_uri(conanfile, url, file_path, md5=None, sha1=None, sha256=None):
file_origin = _path_from_file_uri(url)
shutil.copyfile(file_origin, file_path)

if md5:
check_md5(conanfile, file_path, md5)
if sha1:
check_sha1(conanfile, file_path, sha1)
if sha256:
check_sha256(conanfile, file_path, sha256)

def _path_from_file_uri(uri):
path = urlparse(uri).path
return url2pathname(path)


def rename(conanfile, src, dst):
"""
Expand Down
27 changes: 27 additions & 0 deletions conans/test/unittests/tools/files/test_downloads.py
@@ -1,4 +1,5 @@
import os
import platform

import pytest
import requests
Expand Down Expand Up @@ -157,6 +158,32 @@ def test_download_no_retries_errors(self, bottle_server):
assert "Waiting" not in str(conanfile.output)
assert "retry" not in str(conanfile.output)

def test_download_localfile(self):
conanfile = ConanFileMock()
conanfile._conan_requester = requests

file_location = os.path.join(temp_folder(), "file.txt")
save(file_location, "this is some content")

file_url = f"file:///{file_location}"
file_md5 = "736db904ad222bf88ee6b8d103fceb8e"

dest = os.path.join(temp_folder(), "downloaded_file.txt")
download(conanfile, file_url, dest, md5=file_md5)
content = load(dest)
assert "this is some content" == content

def test_download_localfile_notfound(self):
conanfile = ConanFileMock()
conanfile._conan_requester = requests

file_url = "file:///path/to/missing/file.txt"
dest = os.path.join(temp_folder(), "file.txt")

with pytest.raises(FileNotFoundError) as exc:
download(conanfile, file_url, dest)

assert "No such file" in str(exc.value)

@pytest.fixture()
def bottle_server_zip():
Expand Down

0 comments on commit b20b72b

Please sign in to comment.