Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ability to reference files in the local filesystem with file:// s… #11569

Merged
Merged
28 changes: 24 additions & 4 deletions conan/tools/files/files.py
Expand Up @@ -11,6 +11,8 @@
from fnmatch import fnmatch

import six
from urllib.parse import urlparse
from urllib.request import url2pathname

from conan.tools import CONAN_TOOLCHAIN_ARGS_FILE, CONAN_TOOLCHAIN_ARGS_SECTION
from conan.tools.apple.apple import is_apple_os
Expand Down Expand Up @@ -165,10 +167,13 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None,

def _download_file(file_url):
# The download cache is only used if a checksum is provided, otherwise, a normal download
run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache,
user_download=True, url=file_url,
file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite,
auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256)
if file_url.startswith("file:"):
_copy_local_file_from_uri(conanfile, url=file_url, file_path=filename, md5=md5, sha1=sha1, sha256=sha256)
else:
run_downloader(requester=requester, output=out, verify=verify, download_cache=download_cache,
user_download=True, url=file_url,
file_path=filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite,
auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256)
out.writeln("")

if not isinstance(url, (list, tuple)):
Expand All @@ -184,6 +189,21 @@ def _download_file(file_url):
else:
raise ConanException("All downloads from ({}) URLs have failed.".format(len(url)))

def _copy_local_file_from_uri(conanfile, url, file_path, md5=None, sha1=None, sha256=None):
file_origin = _path_from_file_uri(url)
shutil.copyfile(file_origin, file_path)

if md5:
check_md5(conanfile, file_path, md5)
if sha1:
check_sha1(conanfile, file_path, sha1)
if sha256:
check_sha256(conanfile, file_path, sha256)

def _path_from_file_uri(uri):
path = urlparse(uri).path
return url2pathname(path)


def rename(conanfile, src, dst):
"""
Expand Down
28 changes: 28 additions & 0 deletions conans/test/unittests/tools/files/test_downloads.py
@@ -1,4 +1,5 @@
import os
import platform

import pytest
import requests
Expand Down Expand Up @@ -157,6 +158,33 @@ def test_download_no_retries_errors(self, bottle_server):
assert "Waiting" not in str(conanfile.output)
assert "retry" not in str(conanfile.output)

def test_download_localfile(self):
conanfile = ConanFileMock()
conanfile._conan_requester = requests

file_location = os.path.join(temp_folder(), "file.txt")
with open(file_location, 'w') as textfile:
textfile.write('this is some content\n')
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
with open(file_location, 'w') as textfile:
textfile.write('this is some content\n')
save(file_location, "this is some content\n")

We have one line helpers for this.


file_url = f"file:///{file_location}"
file_md5 = "a0b156435474e688206c68e5c66a3327"

dest = os.path.join(temp_folder(), "downloaded_file.txt")
download(conanfile, file_url, dest, md5=file_md5)
content = load(dest)
assert "this is some content" == content.rstrip()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
assert "this is some content" == content.rstrip()
assert "this is some content" == content

The rstrip() is probably unnecessary, and risks hiding a bug somewhere. The file must be identical

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed it by changing the md5 that is checked above (as it stands, given the md5 check, this is probably redundant).

Issue was that most command line text editors and the shell itself will append a newline character, so all of my attempts to get the md5 sum of a string will give me the md5 sum of a string ending with the newline character. Given that a few lines above we have control of the exact byte-wise contents of the file, I think it's safe to remove the rstrip()


def test_download_localfile_notfound(self):
conanfile = ConanFileMock()
conanfile._conan_requester = requests

file_url = "file:///path/to/missing/file.txt"
dest = os.path.join(temp_folder(), "file.txt")

with pytest.raises(FileNotFoundError) as exc:
download(conanfile, file_url, dest)

assert "No such file" in str(exc.value)

@pytest.fixture()
def bottle_server_zip():
Expand Down