New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add ability to reference files in the local filesystem with file:// s… #11569
Changes from 3 commits
866417b
a182818
130b588
9b7f613
cf24294
db618c9
c3f820e
4e8508b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,11 +1,14 @@ | ||
from conans.client.downloaders.cached_file_downloader import CachedFileDownloader | ||
from conans.client.downloaders.file_downloader import FileDownloader | ||
from conans.client.downloaders.local_file_downloader import LocalFileDownloader | ||
|
||
|
||
def run_downloader(requester, output, verify, retry, retry_wait, download_cache, user_download=False, | ||
**kwargs): | ||
def run_downloader(requester, output, verify, retry, retry_wait, download_cache, local_filesystem, | ||
user_download=False, **kwargs): | ||
downloader = FileDownloader(requester=requester, output=output, verify=verify, | ||
config_retry=retry, config_retry_wait=retry_wait) | ||
if download_cache: | ||
if local_filesystem: | ||
downloader = LocalFileDownloader(output=output) | ||
elif download_cache: | ||
downloader = CachedFileDownloader(download_cache, downloader, user_download=user_download) | ||
return downloader.download(**kwargs) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
import os | ||
from urllib.parse import urlparse | ||
from urllib.request import url2pathname | ||
from shutil import copyfile | ||
|
||
from conans.client.tools.files import check_md5, check_sha1, check_sha256 | ||
from conans.errors import ConanException | ||
|
||
|
||
class LocalFileDownloader(object): | ||
|
||
def __init__(self, output): | ||
self._output = output | ||
|
||
def download(self, url, file_path, md5=None, sha1=None, sha256=None, **kwargs): | ||
|
||
file_origin = self._path_from_file_uri(url) | ||
copyfile(file_origin, file_path) | ||
|
||
if md5: | ||
check_md5(file_path, md5) | ||
if sha1: | ||
check_sha1(file_path, sha1) | ||
if sha256: | ||
check_sha256(file_path, sha256) | ||
|
||
def _path_from_file_uri(self, uri): | ||
path = urlparse(uri).path | ||
return url2pathname(path) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,7 @@ | ||
pytest>=6.1.1, <7.0.0; python_version > '3.0' | ||
pytest>=4.6.11; python_version < '3.0' | ||
pytest-xdist # To launch in N cores with pytest -n | ||
pyfakefs>=4.5.6 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'd say definitely not worth the complexities and problems for using more third parties, for this very specific use case. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Sorry, I didn't realize it was There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yup, just used for testing! I exposes an in-memory filesystem to tests such that python functions that operate on the filesystem read from this instead. For the purposes of this test, writing a temp file on the actual file system and reading back is also an option. I wanted to a be a little more explicit making sure that paths like There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'd say yes, we already do a ton of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done! Still does the job :D |
||
parameterized>=0.6.3 | ||
mock>=1.3.0, <1.4.0 | ||
WebTest>=2.0.18, <2.1.0 | ||
|
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -1,4 +1,5 @@ | ||||||
import os | ||||||
import platform | ||||||
|
||||||
import pytest | ||||||
import requests | ||||||
|
@@ -157,6 +158,33 @@ def test_download_no_retries_errors(self, bottle_server): | |||||
assert "Waiting" not in str(conanfile.output) | ||||||
assert "retry" not in str(conanfile.output) | ||||||
|
||||||
def test_download_localfile(self, fs): | ||||||
conanfile = ConanFileMock() | ||||||
conanfile._conan_requester = requests | ||||||
|
||||||
file_location = '/path/to/file.txt' | ||||||
if platform.system() == "Windows": | ||||||
file_location = "C:" + file_location | ||||||
fs.create_file(file_location, contents=b'this is some content\n') | ||||||
file_url = f"file:///{file_location}" | ||||||
file_md5 = "a0b156435474e688206c68e5c66a3327" | ||||||
|
||||||
dest = os.path.join(temp_folder(), "file.txt") | ||||||
download(conanfile, file_url, dest, md5=file_md5) | ||||||
content = load(dest) | ||||||
assert "this is some content" == content.rstrip() | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
The rstrip() is probably unnecessary, and risks hiding a bug somewhere. The file must be identical There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed it by changing the md5 that is checked above (as it stands, given the md5 check, this is probably redundant). Issue was that most command line text editors and the shell itself will append a newline character, so all of my attempts to get the md5 sum of a string will give me the md5 sum of a string ending with the newline character. Given that a few lines above we have control of the exact byte-wise contents of the file, I think it's safe to remove the |
||||||
|
||||||
def test_download_localfile_notfound(self): | ||||||
conanfile = ConanFileMock() | ||||||
conanfile._conan_requester = requests | ||||||
|
||||||
file_url = "file:///path/to/missing/file.txt" | ||||||
dest = os.path.join(temp_folder(), "file.txt") | ||||||
|
||||||
with pytest.raises(FileNotFoundError) as exc: | ||||||
download(conanfile, file_url, dest) | ||||||
|
||||||
assert "No such file" in str(exc.value) | ||||||
|
||||||
@pytest.fixture() | ||||||
def bottle_server_zip(): | ||||||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Note here:
shutil.copyfile()
will overwrite the file if it already exists - I've not added any logic to handle that case. From what I can see, in the newer implementation we are defaulting tooverwrite=True
always, and theoverwrite
argument is no longer exposed.See here: https://github.com/conan-io/conan/blob/develop/conan/tools/files/files.py#L151
But I can see the file downloader does interpret this flag:
https://github.com/conan-io/conan/blob/develop/conans/client/downloaders/file_downloader.py#L46-L53
If there are plans to make overwrites configurable (via
conf
like with the other options), then it's worth implementing now