Store checksum of downloaded (and uploaded) packages (#5910)

* Added required = True to subparsers in order to print error message in Py2 and Py3.

* sync

* basic concurrent upload at reference level with futures

* revert changes

* add line

* Lock buggy urllib3 (#5808)

* app simplifying (#5806)

* Apply lockfile before updating downstream requires (#5771)

* apply graph_lock before looking for overrides

* first step: get rid of the warning

* cleaner if graph_lock is passed to the function

* only update requires upstream if no lockfile is applied

* fix tests

* Deprecation of CONAN_USERNAME and CONAN_CHANNEL: fix error message (#5756)

* if CONAN_USERNAME and CONAN_CHANNEL are deprecated, the error cannot recommend them

* update tests accordingly

* test client load() file method (#5815)

* no user/channel repr without _ (#5817)

* no user/channel repr without _

* minor fixes

* fix tests

* Remove py34 (#5820)

* fix upload package id (#5824)

* - update macOS, watchOS, tvOS, iOS version numbers (#5823)

* Refresh token client support.  (#5662)

* Refresh token client support. Missing tests. Missing migration

* public method

* WIP

* Refresh almost there

* Removed prints

* Try migrate

* Migration

* Add comment

* Refresh token flow following RFC recommentations

* Refresh ok

* review

* Remove traces

* Refactor capabilities

* Removed tmp file

* Review

* #5819 Show warning message for Python 3.4 (#5829)

* #5819 Show warning message for Python 3.4

- Add new warning message for python 3.4 which is no longer supported
- Added funcional tests to validate both python 3.4 and 2.x

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* #5819 Fix broken tests

Signed-off-by: Uilian Ries <uilianries@gmail.com>

* Add cpp_info.name to cmake and pkg_config generators (#5598)

* Add cpp_info.name to cmake generators

* Fix unit tests to mimic real behavior

* cmake_paths test

* add test for cmake generator

* Add cmake_find_package test

* fix test in py3

* Applied cpp_info.name to pkg_config generator

* check different name in pkg_config

* sync with develop

* store files checksum

* store downloaded recipes checksums

* store downloaded package checksum

* add checksums to metadata

* use checksums property

* refactor

* change function nae

* refactor

* minor changes

* change names

* minor changes

* minor changes

* fix comment

* minor changes

* minor changes

* test checksums in unit tests

* check checksums metatada

* return empty dict if not found

* move code

* check downloaded metadata

* change var name

* check real checksum

conans/client/cmd/uploader.py

@@ -5,7 +5,7 @@
 from collections import defaultdict
 
 from conans.util import progress_bar
-from conans.client.remote_manager import is_package_snapshot_complete
+from conans.client.remote_manager import is_package_snapshot_complete, calc_files_checksum
 from conans.client.source import complete_recipe_sources
 from conans.errors import ConanException, NotFoundException
 from conans.model.manifest import gather_files, FileTreeManifest
@@ -220,6 +220,10 @@ def _upload_recipe(self, ref, conanfile, retry, retry_wait, policy, remote, remo
 
         t1 = time.time()
         the_files = self._compress_recipe_files(ref)
+
+        with self._cache.package_layout(ref).update_metadata() as metadata:
+            metadata.recipe.checksums = calc_files_checksum(the_files)
+
         local_manifest = FileTreeManifest.loads(load(the_files["conanmanifest.txt"]))
 
         remote_manifest = None
@@ -273,6 +277,10 @@ def _upload_package(self, pref, retry=None, retry_wait=None, integrity_check=Fal
 
         t1 = time.time()
         the_files = self._compress_package_files(pref, integrity_check)
+
+        with self._cache.package_layout(pref.ref).update_metadata() as metadata:
+            metadata.packages[pref.id].checksums = calc_files_checksum(the_files)
+
         if policy == UPLOAD_POLICY_SKIP:
             return None
         files_to_upload, deleted = self._package_files_to_upload(pref, policy, the_files, p_remote)
@@ -316,6 +324,7 @@ def _compress_recipe_files(self, ref):
         src_files, src_symlinks = gather_files(export_src_folder)
         the_files = _compress_recipe_files(files, symlinks, src_files, src_symlinks, export_folder,
                                            self._output)
+
         return the_files
 
     def _compress_package_files(self, pref, integrity_check):

conans/client/remote_manager.py

@@ -15,7 +15,7 @@
 from conans.util import progress_bar
 from conans.util.env_reader import get_env
 from conans.util.files import make_read_only, mkdir, rmdir, tar_extract, touch_folder, \
-    merge_directories
+    merge_directories, md5sum, sha1sum
 from conans.util.log import logger
 # FIXME: Eventually, when all output is done, tracer functions should be moved to the recorder class
 from conans.util.tracer import (log_package_download,
@@ -87,6 +87,8 @@ def get_recipe(self, ref, remote):
         duration = time.time() - t1
         log_recipe_download(ref, duration, remote.name, zipped_files)
 
+        recipe_checksums = calc_files_checksum(zipped_files)
+
         unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME, output=self._output)
         # Make sure that the source dir is deleted
         package_layout = self._cache.package_layout(ref)
@@ -96,6 +98,7 @@ def get_recipe(self, ref, remote):
 
         with package_layout.update_metadata() as metadata:
             metadata.recipe.revision = ref.revision
+            metadata.recipe.checksums = recipe_checksums
 
         self._hook_manager.execute("post_download_recipe", conanfile_path=conanfile_path,
                                    reference=ref, remote=remote)
@@ -138,9 +141,12 @@ def get_package(self, pref, dest_folder, remote, output, recorder):
                 raise PackageNotFoundException(pref)
             zipped_files = self._call_remote(remote, "get_package", pref, dest_folder)
 
+            package_checksums = calc_files_checksum(zipped_files)
+
             with self._cache.package_layout(pref.ref).update_metadata() as metadata:
                 metadata.packages[pref.id].revision = pref.revision
                 metadata.packages[pref.id].recipe_revision = pref.ref.revision
+                metadata.packages[pref.id].checksums = package_checksums
 
             duration = time.time() - t1
             log_package_download(pref, duration, remote, zipped_files)
@@ -248,6 +254,10 @@ def _call_remote(self, remote, method, *argc, **argv):
             raise ConanException(exc, remote=remote)
 
 
+def calc_files_checksum(files):
+    return {file_name: {"md5": md5sum(path), "sha1": sha1sum(path)} for file_name, path in files.items()}
+
+
 def is_package_snapshot_complete(snapshot):
     integrity = True
     for keyword in ["conaninfo", "conanmanifest", "conan_package"]:

conans/model/package_metadata.py

@@ -9,6 +9,7 @@ class _RecipeMetadata(object):
     def __init__(self):
         self._revision = None
         self.properties = {}
+        self.checksums = {}
         self.remote = None
 
     @property
@@ -22,7 +23,8 @@ def revision(self, r):
     def to_dict(self):
         ret = {"revision": self.revision,
                "remote": self.remote,
-               "properties": self.properties}
+               "properties": self.properties,
+               "checksums": self.checksums}
         return ret
 
     @staticmethod
@@ -31,6 +33,7 @@ def loads(data):
         ret.revision = data["revision"]
         ret.remote = data.get("remote")
         ret.properties = data["properties"]
+        ret.checksums = data.get("checksums", {})
         ret.time = data.get("time")
         return ret
 
@@ -41,6 +44,7 @@ def __init__(self):
         self._revision = None
         self._recipe_revision = None
         self.properties = {}
+        self.checksums = {}
         self.remote = None
 
     @property
@@ -63,7 +67,8 @@ def to_dict(self):
         ret = {"revision": self.revision,
                "recipe_revision": self.recipe_revision,
                "remote": self.remote,
-               "properties": self.properties}
+               "properties": self.properties,
+               "checksums": self.checksums}
         return ret
 
     @staticmethod
@@ -72,6 +77,7 @@ def loads(data):
         ret.revision = data.get("revision")
         ret.recipe_revision = data.get("recipe_revision")
         ret.properties = data.get("properties")
+        ret.checksums = data.get("checksums", {})
         ret.remote = data.get("remote")
         return ret
 

conans/test/functional/command/upload_test.py

@@ -749,6 +749,34 @@ def upload_without_user_channel_test(self):
         self.assertIn("Uploading package 1/1: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 to 'default'",
                       client.out)
 
+    def checksums_metadata_test(self):
+        client = TestClient(default_server_user=True)
+        client.save({"conanfile.py": GenConanfile()})
+        client.run('create . lib/1.0@user/channel')
+        client.run('upload lib/1.0 -c --all -r default')
+        ref = ConanFileReference("lib", "1.0", "user", "channel")
+        metadata = client.cache.package_layout(ref).load_metadata()
+        package_md5 = metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"]["md5"]
+        package_sha1 = metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"][
+            "sha1"]
+        recipe_md5 = metadata.recipe.checksums["conanfile.py"]["md5"]
+        recipe_sha1 = metadata.recipe.checksums["conanfile.py"]["sha1"]
+        self.assertEqual(package_md5, "25f53ac9685e07815b990e7c6f21bfd0")
+        self.assertEqual(package_sha1, "be152c82859285658a06816aaaec529a159c33d3")
+        self.assertEqual(recipe_md5, "8eda41e0997f3eacc436fb6c621d7396")
+        self.assertEqual(recipe_sha1, "b97d6b26be5bd02252a44c265755f873cf5ec70b")
+        client.run('remove * -f')
+        client.run('install lib/1.0@user/channel -r default')
+        metadata = client.cache.package_layout(ref).load_metadata()
+        self.assertEqual(
+            metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"]["md5"],
+            package_md5)
+        self.assertEqual(
+            metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"]["sha1"],
+            package_sha1)
+        self.assertEqual(metadata.recipe.checksums["conanfile.py"]["md5"], recipe_md5)
+        self.assertEqual(metadata.recipe.checksums["conanfile.py"]["sha1"], recipe_sha1)
+
     def upload_without_cleaned_user_test(self):
         """ When a user is not authenticated, uploads failed first time
         https://github.com/conan-io/conan/issues/5878

conans/test/unittests/model/package_metadata_test.py

@@ -8,16 +8,24 @@ class PackageMetadataTest(unittest.TestCase):
     def test_load_unload(self):
         a = PackageMetadata()
         a.recipe.revision = "rev"
+        a.recipe.checksums["somefile1"] = {"md5": "50b2137a5d63567b7e88b743a3b594cf",
+                                           "sha1": "0b7e8ed59ff4eacb95fd3cc8e17a8034584a96c2"}
         a.packages["ID"].recipe_revision = "rec_rev"
         a.packages["ID"].revision = "revp"
         a.packages["ID"].properties["Someprop"] = "23"
+        a.packages["ID"].checksums["somefile2"] = {"md5": "efb7597b146344532fe8da2b79860aaa",
+                                                   "sha1": "cc3e6eae41eca26538630f4cd5b0bf4fb52e2d"}
 
         tmp = a.dumps()
 
         b = PackageMetadata.loads(tmp)
 
         self.assertEqual(b, a)
         self.assertEqual(b.packages["ID"].properties["Someprop"], "23")
+        self.assertEqual(b.recipe.checksums["somefile1"]["md5"],
+                         "50b2137a5d63567b7e88b743a3b594cf")
+        self.assertEqual(b.packages["ID"].checksums["somefile2"]["sha1"],
+                         "cc3e6eae41eca26538630f4cd5b0bf4fb52e2d")
 
     def test_other_types_than_str(self):
         a = PackageMetadata()