diff --git a/conans/client/cmd/uploader.py b/conans/client/cmd/uploader.py index 3a0d1ada2a2..2e9bb1c6011 100644 --- a/conans/client/cmd/uploader.py +++ b/conans/client/cmd/uploader.py @@ -5,7 +5,7 @@ from collections import defaultdict from conans.util import progress_bar -from conans.client.remote_manager import is_package_snapshot_complete +from conans.client.remote_manager import is_package_snapshot_complete, calc_files_checksum from conans.client.source import complete_recipe_sources from conans.errors import ConanException, NotFoundException from conans.model.manifest import gather_files, FileTreeManifest @@ -220,6 +220,10 @@ def _upload_recipe(self, ref, conanfile, retry, retry_wait, policy, remote, remo t1 = time.time() the_files = self._compress_recipe_files(ref) + + with self._cache.package_layout(ref).update_metadata() as metadata: + metadata.recipe.checksums = calc_files_checksum(the_files) + local_manifest = FileTreeManifest.loads(load(the_files["conanmanifest.txt"])) remote_manifest = None @@ -273,6 +277,10 @@ def _upload_package(self, pref, retry=None, retry_wait=None, integrity_check=Fal t1 = time.time() the_files = self._compress_package_files(pref, integrity_check) + + with self._cache.package_layout(pref.ref).update_metadata() as metadata: + metadata.packages[pref.id].checksums = calc_files_checksum(the_files) + if policy == UPLOAD_POLICY_SKIP: return None files_to_upload, deleted = self._package_files_to_upload(pref, policy, the_files, p_remote) @@ -316,6 +324,7 @@ def _compress_recipe_files(self, ref): src_files, src_symlinks = gather_files(export_src_folder) the_files = _compress_recipe_files(files, symlinks, src_files, src_symlinks, export_folder, self._output) + return the_files def _compress_package_files(self, pref, integrity_check): diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index d6961ad155d..378fda4f770 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -15,7 +15,7 @@ from conans.util import progress_bar from conans.util.env_reader import get_env from conans.util.files import make_read_only, mkdir, rmdir, tar_extract, touch_folder, \ - merge_directories + merge_directories, md5sum, sha1sum from conans.util.log import logger # FIXME: Eventually, when all output is done, tracer functions should be moved to the recorder class from conans.util.tracer import (log_package_download, @@ -87,6 +87,8 @@ def get_recipe(self, ref, remote): duration = time.time() - t1 log_recipe_download(ref, duration, remote.name, zipped_files) + recipe_checksums = calc_files_checksum(zipped_files) + unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME, output=self._output) # Make sure that the source dir is deleted package_layout = self._cache.package_layout(ref) @@ -96,6 +98,7 @@ def get_recipe(self, ref, remote): with package_layout.update_metadata() as metadata: metadata.recipe.revision = ref.revision + metadata.recipe.checksums = recipe_checksums self._hook_manager.execute("post_download_recipe", conanfile_path=conanfile_path, reference=ref, remote=remote) @@ -138,9 +141,12 @@ def get_package(self, pref, dest_folder, remote, output, recorder): raise PackageNotFoundException(pref) zipped_files = self._call_remote(remote, "get_package", pref, dest_folder) + package_checksums = calc_files_checksum(zipped_files) + with self._cache.package_layout(pref.ref).update_metadata() as metadata: metadata.packages[pref.id].revision = pref.revision metadata.packages[pref.id].recipe_revision = pref.ref.revision + metadata.packages[pref.id].checksums = package_checksums duration = time.time() - t1 log_package_download(pref, duration, remote, zipped_files) @@ -248,6 +254,10 @@ def _call_remote(self, remote, method, *argc, **argv): raise ConanException(exc, remote=remote) +def calc_files_checksum(files): + return {file_name: {"md5": md5sum(path), "sha1": sha1sum(path)} for file_name, path in files.items()} + + def is_package_snapshot_complete(snapshot): integrity = True for keyword in ["conaninfo", "conanmanifest", "conan_package"]: diff --git a/conans/model/package_metadata.py b/conans/model/package_metadata.py index 35e498697de..63afe218896 100644 --- a/conans/model/package_metadata.py +++ b/conans/model/package_metadata.py @@ -9,6 +9,7 @@ class _RecipeMetadata(object): def __init__(self): self._revision = None self.properties = {} + self.checksums = {} self.remote = None @property @@ -22,7 +23,8 @@ def revision(self, r): def to_dict(self): ret = {"revision": self.revision, "remote": self.remote, - "properties": self.properties} + "properties": self.properties, + "checksums": self.checksums} return ret @staticmethod @@ -31,6 +33,7 @@ def loads(data): ret.revision = data["revision"] ret.remote = data.get("remote") ret.properties = data["properties"] + ret.checksums = data.get("checksums", {}) ret.time = data.get("time") return ret @@ -41,6 +44,7 @@ def __init__(self): self._revision = None self._recipe_revision = None self.properties = {} + self.checksums = {} self.remote = None @property @@ -63,7 +67,8 @@ def to_dict(self): ret = {"revision": self.revision, "recipe_revision": self.recipe_revision, "remote": self.remote, - "properties": self.properties} + "properties": self.properties, + "checksums": self.checksums} return ret @staticmethod @@ -72,6 +77,7 @@ def loads(data): ret.revision = data.get("revision") ret.recipe_revision = data.get("recipe_revision") ret.properties = data.get("properties") + ret.checksums = data.get("checksums", {}) ret.remote = data.get("remote") return ret diff --git a/conans/test/functional/command/upload_test.py b/conans/test/functional/command/upload_test.py index 8684119b5ef..5925d54f540 100644 --- a/conans/test/functional/command/upload_test.py +++ b/conans/test/functional/command/upload_test.py @@ -749,6 +749,34 @@ def upload_without_user_channel_test(self): self.assertIn("Uploading package 1/1: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 to 'default'", client.out) + def checksums_metadata_test(self): + client = TestClient(default_server_user=True) + client.save({"conanfile.py": GenConanfile()}) + client.run('create . lib/1.0@user/channel') + client.run('upload lib/1.0 -c --all -r default') + ref = ConanFileReference("lib", "1.0", "user", "channel") + metadata = client.cache.package_layout(ref).load_metadata() + package_md5 = metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"]["md5"] + package_sha1 = metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"][ + "sha1"] + recipe_md5 = metadata.recipe.checksums["conanfile.py"]["md5"] + recipe_sha1 = metadata.recipe.checksums["conanfile.py"]["sha1"] + self.assertEqual(package_md5, "25f53ac9685e07815b990e7c6f21bfd0") + self.assertEqual(package_sha1, "be152c82859285658a06816aaaec529a159c33d3") + self.assertEqual(recipe_md5, "8eda41e0997f3eacc436fb6c621d7396") + self.assertEqual(recipe_sha1, "b97d6b26be5bd02252a44c265755f873cf5ec70b") + client.run('remove * -f') + client.run('install lib/1.0@user/channel -r default') + metadata = client.cache.package_layout(ref).load_metadata() + self.assertEqual( + metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"]["md5"], + package_md5) + self.assertEqual( + metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"]["sha1"], + package_sha1) + self.assertEqual(metadata.recipe.checksums["conanfile.py"]["md5"], recipe_md5) + self.assertEqual(metadata.recipe.checksums["conanfile.py"]["sha1"], recipe_sha1) + def upload_without_cleaned_user_test(self): """ When a user is not authenticated, uploads failed first time https://github.com/conan-io/conan/issues/5878 diff --git a/conans/test/unittests/model/package_metadata_test.py b/conans/test/unittests/model/package_metadata_test.py index fe1f4cde215..ba8e743b22e 100644 --- a/conans/test/unittests/model/package_metadata_test.py +++ b/conans/test/unittests/model/package_metadata_test.py @@ -8,9 +8,13 @@ class PackageMetadataTest(unittest.TestCase): def test_load_unload(self): a = PackageMetadata() a.recipe.revision = "rev" + a.recipe.checksums["somefile1"] = {"md5": "50b2137a5d63567b7e88b743a3b594cf", + "sha1": "0b7e8ed59ff4eacb95fd3cc8e17a8034584a96c2"} a.packages["ID"].recipe_revision = "rec_rev" a.packages["ID"].revision = "revp" a.packages["ID"].properties["Someprop"] = "23" + a.packages["ID"].checksums["somefile2"] = {"md5": "efb7597b146344532fe8da2b79860aaa", + "sha1": "cc3e6eae41eca26538630f4cd5b0bf4fb52e2d"} tmp = a.dumps() @@ -18,6 +22,10 @@ def test_load_unload(self): self.assertEqual(b, a) self.assertEqual(b.packages["ID"].properties["Someprop"], "23") + self.assertEqual(b.recipe.checksums["somefile1"]["md5"], + "50b2137a5d63567b7e88b743a3b594cf") + self.assertEqual(b.packages["ID"].checksums["somefile2"]["sha1"], + "cc3e6eae41eca26538630f4cd5b0bf4fb52e2d") def test_other_types_than_str(self): a = PackageMetadata()