From 538c3550b6102e07739fdc3a0d526200fde6d283 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 23 Mar 2021 15:46:10 +0100 Subject: [PATCH] added dirty management in download cache (#8664) * added dirty management in download cache * fix test * added comments in test --- .../downloaders/cached_file_downloader.py | 8 ++++- .../integration/cache/download_cache_test.py | 34 ++++++++++++++++--- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/conans/client/downloaders/cached_file_downloader.py b/conans/client/downloaders/cached_file_downloader.py index 7038d40615b..7a5216a9edd 100644 --- a/conans/client/downloaders/cached_file_downloader.py +++ b/conans/client/downloaders/cached_file_downloader.py @@ -7,7 +7,7 @@ from conans.client.downloaders.file_downloader import check_checksum from conans.errors import ConanException -from conans.util.files import mkdir +from conans.util.files import mkdir, set_dirty, clean_dirty, is_dirty from conans.util.locks import SimpleLock from conans.util.sha import sha256 as sha256_sum @@ -43,9 +43,15 @@ def download(self, url, file_path=None, md5=None, sha1=None, sha256=None, **kwar with self._lock(h): cached_path = os.path.join(self._cache_folder, h) + if is_dirty(cached_path): + if os.path.exists(cached_path): + os.remove(cached_path) + clean_dirty(cached_path) if not os.path.exists(cached_path): + set_dirty(cached_path) self._file_downloader.download(url=url, file_path=cached_path, md5=md5, sha1=sha1, sha256=sha256, **kwargs) + clean_dirty(cached_path) else: # specific check for corrupted cached files, will raise, but do nothing more # user can report it or "rm -rf cache_folder/path/to/file" diff --git a/conans/test/integration/cache/download_cache_test.py b/conans/test/integration/cache/download_cache_test.py index 2a77d7774fd..da36dcba8da 100644 --- a/conans/test/integration/cache/download_cache_test.py +++ b/conans/test/integration/cache/download_cache_test.py @@ -9,10 +9,11 @@ import pytest from conans.client.downloaders.cached_file_downloader import CachedFileDownloader +from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.test_files import temp_folder from conans.test.utils.tools import TestClient, StoppableThreadBottle from conans.util.env_reader import get_env -from conans.util.files import load, save +from conans.util.files import load, save, set_dirty class DownloadCacheTest(unittest.TestCase): @@ -96,6 +97,28 @@ def package(self): self.assertIn("ERROR: md5 signature failed", client.out) self.assertIn("Cached downloaded file corrupted", client.out) + @pytest.mark.skipif(not get_env("TESTING_REVISIONS_ENABLED", False), reason="Only revisions") + def test_dirty_download(self): + # https://github.com/conan-io/conan/issues/8578 + client = TestClient(default_server_user=True) + cache_folder = temp_folder() + client.run('config set storage.download_cache="%s"' % cache_folder) + client.save({"conanfile.py": GenConanfile().with_package_file("file.txt", "content")}) + client.run("create . pkg/0.1@") + client.run("upload * --all -c") + client.run("remove * -f") + client.run("install pkg/0.1@") + for f in os.listdir(cache_folder): + # damage the file + path = os.path.join(cache_folder, f) + if os.path.isfile(path): + save(path, "broken!") + set_dirty(path) + + client.run("remove * -f") + client.run("install pkg/0.1@") + assert "pkg/0.1: Downloaded package" in client.out + def test_user_downloads_cached(self): http_server = StoppableThreadBottle() @@ -126,7 +149,8 @@ def source(self): self.assertIn("ConanException: md5 signature failed for", client.out) self.assertIn("Provided signature: kk", client.out) self.assertIn("Computed signature: 9893532233caff98cd083a116b013c0b", client.out) - self.assertEqual(1, len(os.listdir(cache_folder))) # Nothing was cached + # There are 2 things in the cache, the "locks" folder and the .dirty file because failure + self.assertEqual(2, len(os.listdir(cache_folder))) # Nothing was cached # This is the right checksum conanfile = textwrap.dedent(""" @@ -148,7 +172,8 @@ def source(self): self.assertEqual("some query", client.load("myfile2.txt")) # 2 files cached, plus "locks" folder = 3 - self.assertEqual(3, len(os.listdir(cache_folder))) + # "locks" folder + 2 files cached + .dirty file from previous failure + self.assertEqual(4, len(os.listdir(cache_folder))) # remove remote file os.remove(file_path) @@ -169,7 +194,8 @@ def source(self): self.assertIn("ERROR: conanfile.py: Error in source() method, line 7", client.out) self.assertIn("Not found: http://localhost", client.out) - @pytest.mark.skipif(get_env("TESTING_REVISIONS_ENABLED", False), reason="Hybrid test with both v1 and v2") + @pytest.mark.skipif(get_env("TESTING_REVISIONS_ENABLED", False), + reason="Hybrid test with both v1 and v2") def test_revision0_v2_skip(self): client = TestClient(default_server_user=True) client.run("config set general.revisions_enabled=False")