From 2d516341777df37064337f4cd72b89931f9eb4a0 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 17 Mar 2021 18:15:50 +0100 Subject: [PATCH 1/3] added dirty management in download cache --- .../downloaders/cached_file_downloader.py | 8 +++++- .../integration/cache/download_cache_test.py | 28 +++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/conans/client/downloaders/cached_file_downloader.py b/conans/client/downloaders/cached_file_downloader.py index 7038d40615b..7a5216a9edd 100644 --- a/conans/client/downloaders/cached_file_downloader.py +++ b/conans/client/downloaders/cached_file_downloader.py @@ -7,7 +7,7 @@ from conans.client.downloaders.file_downloader import check_checksum from conans.errors import ConanException -from conans.util.files import mkdir +from conans.util.files import mkdir, set_dirty, clean_dirty, is_dirty from conans.util.locks import SimpleLock from conans.util.sha import sha256 as sha256_sum @@ -43,9 +43,15 @@ def download(self, url, file_path=None, md5=None, sha1=None, sha256=None, **kwar with self._lock(h): cached_path = os.path.join(self._cache_folder, h) + if is_dirty(cached_path): + if os.path.exists(cached_path): + os.remove(cached_path) + clean_dirty(cached_path) if not os.path.exists(cached_path): + set_dirty(cached_path) self._file_downloader.download(url=url, file_path=cached_path, md5=md5, sha1=sha1, sha256=sha256, **kwargs) + clean_dirty(cached_path) else: # specific check for corrupted cached files, will raise, but do nothing more # user can report it or "rm -rf cache_folder/path/to/file" diff --git a/conans/test/integration/cache/download_cache_test.py b/conans/test/integration/cache/download_cache_test.py index 2a77d7774fd..854c1271154 100644 --- a/conans/test/integration/cache/download_cache_test.py +++ b/conans/test/integration/cache/download_cache_test.py @@ -9,10 +9,11 @@ import pytest from conans.client.downloaders.cached_file_downloader import CachedFileDownloader +from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.test_files import temp_folder from conans.test.utils.tools import TestClient, StoppableThreadBottle from conans.util.env_reader import get_env -from conans.util.files import load, save +from conans.util.files import load, save, set_dirty class DownloadCacheTest(unittest.TestCase): @@ -96,6 +97,28 @@ def package(self): self.assertIn("ERROR: md5 signature failed", client.out) self.assertIn("Cached downloaded file corrupted", client.out) + @pytest.mark.skipif(not get_env("TESTING_REVISIONS_ENABLED", False), reason="Only revisions") + def test_dirty_download(self): + # https://github.com/conan-io/conan/issues/8578 + client = TestClient(default_server_user=True) + cache_folder = temp_folder() + client.run('config set storage.download_cache="%s"' % cache_folder) + client.save({"conanfile.py": GenConanfile().with_package_file("file.txt", "content")}) + client.run("create . pkg/0.1@") + client.run("upload * --all -c") + client.run("remove * -f") + client.run("install pkg/0.1@") + for f in os.listdir(cache_folder): + # damage the file + path = os.path.join(cache_folder, f) + if os.path.isfile(path): + save(path, "broken!") + set_dirty(path) + + client.run("remove * -f") + client.run("install pkg/0.1@") + assert "pkg/0.1: Downloaded package" in client.out + def test_user_downloads_cached(self): http_server = StoppableThreadBottle() @@ -169,7 +192,8 @@ def source(self): self.assertIn("ERROR: conanfile.py: Error in source() method, line 7", client.out) self.assertIn("Not found: http://localhost", client.out) - @pytest.mark.skipif(get_env("TESTING_REVISIONS_ENABLED", False), reason="Hybrid test with both v1 and v2") + @pytest.mark.skipif(get_env("TESTING_REVISIONS_ENABLED", False), + reason="Hybrid test with both v1 and v2") def test_revision0_v2_skip(self): client = TestClient(default_server_user=True) client.run("config set general.revisions_enabled=False") From 997eafbdb9750405a39f599d923e3a7804dc6fb2 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 17 Mar 2021 19:00:14 +0100 Subject: [PATCH 2/3] fix test --- conans/test/integration/cache/download_cache_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conans/test/integration/cache/download_cache_test.py b/conans/test/integration/cache/download_cache_test.py index 854c1271154..5efa4a1e8fe 100644 --- a/conans/test/integration/cache/download_cache_test.py +++ b/conans/test/integration/cache/download_cache_test.py @@ -149,7 +149,7 @@ def source(self): self.assertIn("ConanException: md5 signature failed for", client.out) self.assertIn("Provided signature: kk", client.out) self.assertIn("Computed signature: 9893532233caff98cd083a116b013c0b", client.out) - self.assertEqual(1, len(os.listdir(cache_folder))) # Nothing was cached + self.assertEqual(2, len(os.listdir(cache_folder))) # Nothing was cached # This is the right checksum conanfile = textwrap.dedent(""" @@ -171,7 +171,7 @@ def source(self): self.assertEqual("some query", client.load("myfile2.txt")) # 2 files cached, plus "locks" folder = 3 - self.assertEqual(3, len(os.listdir(cache_folder))) + self.assertEqual(4, len(os.listdir(cache_folder))) # remove remote file os.remove(file_path) From de40094b7b74903d76868f574067ec7ae6bdbf90 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 23 Mar 2021 13:57:25 +0100 Subject: [PATCH 3/3] added comments in test --- conans/test/integration/cache/download_cache_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conans/test/integration/cache/download_cache_test.py b/conans/test/integration/cache/download_cache_test.py index 5efa4a1e8fe..da36dcba8da 100644 --- a/conans/test/integration/cache/download_cache_test.py +++ b/conans/test/integration/cache/download_cache_test.py @@ -149,6 +149,7 @@ def source(self): self.assertIn("ConanException: md5 signature failed for", client.out) self.assertIn("Provided signature: kk", client.out) self.assertIn("Computed signature: 9893532233caff98cd083a116b013c0b", client.out) + # There are 2 things in the cache, the "locks" folder and the .dirty file because failure self.assertEqual(2, len(os.listdir(cache_folder))) # Nothing was cached # This is the right checksum @@ -171,6 +172,7 @@ def source(self): self.assertEqual("some query", client.load("myfile2.txt")) # 2 files cached, plus "locks" folder = 3 + # "locks" folder + 2 files cached + .dirty file from previous failure self.assertEqual(4, len(os.listdir(cache_folder))) # remove remote file