From 0063d9abaa43599572afde55181d445c2fa5ec0f Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Mon, 28 Oct 2019 11:42:08 +0100 Subject: [PATCH] Store checksum of downloaded (and uploaded) packages (#5910) * Added required = True to subparsers in order to print error message in Py2 and Py3. * sync * basic concurrent upload at reference level with futures * revert changes * add line * Lock buggy urllib3 (#5808) * app simplifying (#5806) * Apply lockfile before updating downstream requires (#5771) * apply graph_lock before looking for overrides * first step: get rid of the warning * cleaner if graph_lock is passed to the function * only update requires upstream if no lockfile is applied * fix tests * Deprecation of CONAN_USERNAME and CONAN_CHANNEL: fix error message (#5756) * if CONAN_USERNAME and CONAN_CHANNEL are deprecated, the error cannot recommend them * update tests accordingly * test client load() file method (#5815) * no user/channel repr without _ (#5817) * no user/channel repr without _ * minor fixes * fix tests * Remove py34 (#5820) * fix upload package id (#5824) * - update macOS, watchOS, tvOS, iOS version numbers (#5823) * Refresh token client support. (#5662) * Refresh token client support. Missing tests. Missing migration * public method * WIP * Refresh almost there * Removed prints * Try migrate * Migration * Add comment * Refresh token flow following RFC recommentations * Refresh ok * review * Remove traces * Refactor capabilities * Removed tmp file * Review * #5819 Show warning message for Python 3.4 (#5829) * #5819 Show warning message for Python 3.4 - Add new warning message for python 3.4 which is no longer supported - Added funcional tests to validate both python 3.4 and 2.x Signed-off-by: Uilian Ries * #5819 Fix broken tests Signed-off-by: Uilian Ries * Add cpp_info.name to cmake and pkg_config generators (#5598) * Add cpp_info.name to cmake generators * Fix unit tests to mimic real behavior * cmake_paths test * add test for cmake generator * Add cmake_find_package test * fix test in py3 * Applied cpp_info.name to pkg_config generator * check different name in pkg_config * sync with develop * store files checksum * store downloaded recipes checksums * store downloaded package checksum * add checksums to metadata * use checksums property * refactor * change function nae * refactor * minor changes * change names * minor changes * minor changes * fix comment * minor changes * minor changes * test checksums in unit tests * check checksums metatada * return empty dict if not found * move code * check downloaded metadata * change var name * check real checksum --- conans/client/cmd/uploader.py | 11 +++++++- conans/client/remote_manager.py | 12 +++++++- conans/model/package_metadata.py | 10 +++++-- conans/test/functional/command/upload_test.py | 28 +++++++++++++++++++ .../unittests/model/package_metadata_test.py | 8 ++++++ 5 files changed, 65 insertions(+), 4 deletions(-) diff --git a/conans/client/cmd/uploader.py b/conans/client/cmd/uploader.py index 3a0d1ada2a2..2e9bb1c6011 100644 --- a/conans/client/cmd/uploader.py +++ b/conans/client/cmd/uploader.py @@ -5,7 +5,7 @@ from collections import defaultdict from conans.util import progress_bar -from conans.client.remote_manager import is_package_snapshot_complete +from conans.client.remote_manager import is_package_snapshot_complete, calc_files_checksum from conans.client.source import complete_recipe_sources from conans.errors import ConanException, NotFoundException from conans.model.manifest import gather_files, FileTreeManifest @@ -220,6 +220,10 @@ def _upload_recipe(self, ref, conanfile, retry, retry_wait, policy, remote, remo t1 = time.time() the_files = self._compress_recipe_files(ref) + + with self._cache.package_layout(ref).update_metadata() as metadata: + metadata.recipe.checksums = calc_files_checksum(the_files) + local_manifest = FileTreeManifest.loads(load(the_files["conanmanifest.txt"])) remote_manifest = None @@ -273,6 +277,10 @@ def _upload_package(self, pref, retry=None, retry_wait=None, integrity_check=Fal t1 = time.time() the_files = self._compress_package_files(pref, integrity_check) + + with self._cache.package_layout(pref.ref).update_metadata() as metadata: + metadata.packages[pref.id].checksums = calc_files_checksum(the_files) + if policy == UPLOAD_POLICY_SKIP: return None files_to_upload, deleted = self._package_files_to_upload(pref, policy, the_files, p_remote) @@ -316,6 +324,7 @@ def _compress_recipe_files(self, ref): src_files, src_symlinks = gather_files(export_src_folder) the_files = _compress_recipe_files(files, symlinks, src_files, src_symlinks, export_folder, self._output) + return the_files def _compress_package_files(self, pref, integrity_check): diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index d6961ad155d..378fda4f770 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -15,7 +15,7 @@ from conans.util import progress_bar from conans.util.env_reader import get_env from conans.util.files import make_read_only, mkdir, rmdir, tar_extract, touch_folder, \ - merge_directories + merge_directories, md5sum, sha1sum from conans.util.log import logger # FIXME: Eventually, when all output is done, tracer functions should be moved to the recorder class from conans.util.tracer import (log_package_download, @@ -87,6 +87,8 @@ def get_recipe(self, ref, remote): duration = time.time() - t1 log_recipe_download(ref, duration, remote.name, zipped_files) + recipe_checksums = calc_files_checksum(zipped_files) + unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME, output=self._output) # Make sure that the source dir is deleted package_layout = self._cache.package_layout(ref) @@ -96,6 +98,7 @@ def get_recipe(self, ref, remote): with package_layout.update_metadata() as metadata: metadata.recipe.revision = ref.revision + metadata.recipe.checksums = recipe_checksums self._hook_manager.execute("post_download_recipe", conanfile_path=conanfile_path, reference=ref, remote=remote) @@ -138,9 +141,12 @@ def get_package(self, pref, dest_folder, remote, output, recorder): raise PackageNotFoundException(pref) zipped_files = self._call_remote(remote, "get_package", pref, dest_folder) + package_checksums = calc_files_checksum(zipped_files) + with self._cache.package_layout(pref.ref).update_metadata() as metadata: metadata.packages[pref.id].revision = pref.revision metadata.packages[pref.id].recipe_revision = pref.ref.revision + metadata.packages[pref.id].checksums = package_checksums duration = time.time() - t1 log_package_download(pref, duration, remote, zipped_files) @@ -248,6 +254,10 @@ def _call_remote(self, remote, method, *argc, **argv): raise ConanException(exc, remote=remote) +def calc_files_checksum(files): + return {file_name: {"md5": md5sum(path), "sha1": sha1sum(path)} for file_name, path in files.items()} + + def is_package_snapshot_complete(snapshot): integrity = True for keyword in ["conaninfo", "conanmanifest", "conan_package"]: diff --git a/conans/model/package_metadata.py b/conans/model/package_metadata.py index 35e498697de..63afe218896 100644 --- a/conans/model/package_metadata.py +++ b/conans/model/package_metadata.py @@ -9,6 +9,7 @@ class _RecipeMetadata(object): def __init__(self): self._revision = None self.properties = {} + self.checksums = {} self.remote = None @property @@ -22,7 +23,8 @@ def revision(self, r): def to_dict(self): ret = {"revision": self.revision, "remote": self.remote, - "properties": self.properties} + "properties": self.properties, + "checksums": self.checksums} return ret @staticmethod @@ -31,6 +33,7 @@ def loads(data): ret.revision = data["revision"] ret.remote = data.get("remote") ret.properties = data["properties"] + ret.checksums = data.get("checksums", {}) ret.time = data.get("time") return ret @@ -41,6 +44,7 @@ def __init__(self): self._revision = None self._recipe_revision = None self.properties = {} + self.checksums = {} self.remote = None @property @@ -63,7 +67,8 @@ def to_dict(self): ret = {"revision": self.revision, "recipe_revision": self.recipe_revision, "remote": self.remote, - "properties": self.properties} + "properties": self.properties, + "checksums": self.checksums} return ret @staticmethod @@ -72,6 +77,7 @@ def loads(data): ret.revision = data.get("revision") ret.recipe_revision = data.get("recipe_revision") ret.properties = data.get("properties") + ret.checksums = data.get("checksums", {}) ret.remote = data.get("remote") return ret diff --git a/conans/test/functional/command/upload_test.py b/conans/test/functional/command/upload_test.py index 8684119b5ef..5925d54f540 100644 --- a/conans/test/functional/command/upload_test.py +++ b/conans/test/functional/command/upload_test.py @@ -749,6 +749,34 @@ def upload_without_user_channel_test(self): self.assertIn("Uploading package 1/1: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 to 'default'", client.out) + def checksums_metadata_test(self): + client = TestClient(default_server_user=True) + client.save({"conanfile.py": GenConanfile()}) + client.run('create . lib/1.0@user/channel') + client.run('upload lib/1.0 -c --all -r default') + ref = ConanFileReference("lib", "1.0", "user", "channel") + metadata = client.cache.package_layout(ref).load_metadata() + package_md5 = metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"]["md5"] + package_sha1 = metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"][ + "sha1"] + recipe_md5 = metadata.recipe.checksums["conanfile.py"]["md5"] + recipe_sha1 = metadata.recipe.checksums["conanfile.py"]["sha1"] + self.assertEqual(package_md5, "25f53ac9685e07815b990e7c6f21bfd0") + self.assertEqual(package_sha1, "be152c82859285658a06816aaaec529a159c33d3") + self.assertEqual(recipe_md5, "8eda41e0997f3eacc436fb6c621d7396") + self.assertEqual(recipe_sha1, "b97d6b26be5bd02252a44c265755f873cf5ec70b") + client.run('remove * -f') + client.run('install lib/1.0@user/channel -r default') + metadata = client.cache.package_layout(ref).load_metadata() + self.assertEqual( + metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"]["md5"], + package_md5) + self.assertEqual( + metadata.packages[NO_SETTINGS_PACKAGE_ID].checksums["conan_package.tgz"]["sha1"], + package_sha1) + self.assertEqual(metadata.recipe.checksums["conanfile.py"]["md5"], recipe_md5) + self.assertEqual(metadata.recipe.checksums["conanfile.py"]["sha1"], recipe_sha1) + def upload_without_cleaned_user_test(self): """ When a user is not authenticated, uploads failed first time https://github.com/conan-io/conan/issues/5878 diff --git a/conans/test/unittests/model/package_metadata_test.py b/conans/test/unittests/model/package_metadata_test.py index fe1f4cde215..ba8e743b22e 100644 --- a/conans/test/unittests/model/package_metadata_test.py +++ b/conans/test/unittests/model/package_metadata_test.py @@ -8,9 +8,13 @@ class PackageMetadataTest(unittest.TestCase): def test_load_unload(self): a = PackageMetadata() a.recipe.revision = "rev" + a.recipe.checksums["somefile1"] = {"md5": "50b2137a5d63567b7e88b743a3b594cf", + "sha1": "0b7e8ed59ff4eacb95fd3cc8e17a8034584a96c2"} a.packages["ID"].recipe_revision = "rec_rev" a.packages["ID"].revision = "revp" a.packages["ID"].properties["Someprop"] = "23" + a.packages["ID"].checksums["somefile2"] = {"md5": "efb7597b146344532fe8da2b79860aaa", + "sha1": "cc3e6eae41eca26538630f4cd5b0bf4fb52e2d"} tmp = a.dumps() @@ -18,6 +22,10 @@ def test_load_unload(self): self.assertEqual(b, a) self.assertEqual(b.packages["ID"].properties["Someprop"], "23") + self.assertEqual(b.recipe.checksums["somefile1"]["md5"], + "50b2137a5d63567b7e88b743a3b594cf") + self.assertEqual(b.packages["ID"].checksums["somefile2"]["sha1"], + "cc3e6eae41eca26538630f4cd5b0bf4fb52e2d") def test_other_types_than_str(self): a = PackageMetadata()