added dirty management in download cache (#8664)

* added dirty management in download cache

* fix test

* added comments in test

conans/client/downloaders/cached_file_downloader.py

@@ -7,7 +7,7 @@
 
 from conans.client.downloaders.file_downloader import check_checksum
 from conans.errors import ConanException
-from conans.util.files import mkdir
+from conans.util.files import mkdir, set_dirty, clean_dirty, is_dirty
 from conans.util.locks import SimpleLock
 from conans.util.sha import sha256 as sha256_sum
 
@@ -43,9 +43,15 @@ def download(self, url, file_path=None, md5=None, sha1=None, sha256=None, **kwar
 
         with self._lock(h):
             cached_path = os.path.join(self._cache_folder, h)
+            if is_dirty(cached_path):
+                if os.path.exists(cached_path):
+                    os.remove(cached_path)
+                clean_dirty(cached_path)
             if not os.path.exists(cached_path):
+                set_dirty(cached_path)
                 self._file_downloader.download(url=url, file_path=cached_path, md5=md5,
                                                sha1=sha1, sha256=sha256, **kwargs)
+                clean_dirty(cached_path)
             else:
                 # specific check for corrupted cached files, will raise, but do nothing more
                 # user can report it or "rm -rf cache_folder/path/to/file"

conans/test/integration/cache/download_cache_test.py

@@ -9,10 +9,11 @@
 import pytest
 
 from conans.client.downloaders.cached_file_downloader import CachedFileDownloader
+from conans.test.assets.genconanfile import GenConanfile
 from conans.test.utils.test_files import temp_folder
 from conans.test.utils.tools import TestClient, StoppableThreadBottle
 from conans.util.env_reader import get_env
-from conans.util.files import load, save
+from conans.util.files import load, save, set_dirty
 
 
 class DownloadCacheTest(unittest.TestCase):
@@ -96,6 +97,28 @@ def package(self):
         self.assertIn("ERROR: md5 signature failed", client.out)
         self.assertIn("Cached downloaded file corrupted", client.out)
 
+    @pytest.mark.skipif(not get_env("TESTING_REVISIONS_ENABLED", False), reason="Only revisions")
+    def test_dirty_download(self):
+        # https://github.com/conan-io/conan/issues/8578
+        client = TestClient(default_server_user=True)
+        cache_folder = temp_folder()
+        client.run('config set storage.download_cache="%s"' % cache_folder)
+        client.save({"conanfile.py": GenConanfile().with_package_file("file.txt", "content")})
+        client.run("create . pkg/0.1@")
+        client.run("upload * --all -c")
+        client.run("remove * -f")
+        client.run("install pkg/0.1@")
+        for f in os.listdir(cache_folder):
+            # damage the file
+            path = os.path.join(cache_folder, f)
+            if os.path.isfile(path):
+                save(path, "broken!")
+                set_dirty(path)
+
+        client.run("remove * -f")
+        client.run("install pkg/0.1@")
+        assert "pkg/0.1: Downloaded package" in client.out
+
     def test_user_downloads_cached(self):
         http_server = StoppableThreadBottle()
 
@@ -126,7 +149,8 @@ def source(self):
         self.assertIn("ConanException: md5 signature failed for", client.out)
         self.assertIn("Provided signature: kk", client.out)
         self.assertIn("Computed signature: 9893532233caff98cd083a116b013c0b", client.out)
-        self.assertEqual(1, len(os.listdir(cache_folder)))  # Nothing was cached
+        # There are 2 things in the cache, the "locks" folder and the .dirty file because failure
+        self.assertEqual(2, len(os.listdir(cache_folder)))  # Nothing was cached
 
         # This is the right checksum
         conanfile = textwrap.dedent("""
@@ -148,7 +172,8 @@ def source(self):
         self.assertEqual("some query", client.load("myfile2.txt"))
 
         # 2 files cached, plus "locks" folder = 3
-        self.assertEqual(3, len(os.listdir(cache_folder)))
+        # "locks" folder + 2 files cached + .dirty file from previous failure
+        self.assertEqual(4, len(os.listdir(cache_folder)))
 
         # remove remote file
         os.remove(file_path)
@@ -169,7 +194,8 @@ def source(self):
         self.assertIn("ERROR: conanfile.py: Error in source() method, line 7", client.out)
         self.assertIn("Not found: http://localhost", client.out)
 
-    @pytest.mark.skipif(get_env("TESTING_REVISIONS_ENABLED", False), reason="Hybrid test with both v1 and v2")
+    @pytest.mark.skipif(get_env("TESTING_REVISIONS_ENABLED", False),
+                        reason="Hybrid test with both v1 and v2")
     def test_revision0_v2_skip(self):
         client = TestClient(default_server_user=True)
         client.run("config set general.revisions_enabled=False")