From 98a762db13c9245294e9163fd88243df4c530e14 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 11 Mar 2021 16:31:05 -0800 Subject: [PATCH] Dependency scanning (#6) * certbot constraints to reqs * rename dev constraints to reqs * Rename pipstrap constraints to requirements --- snap/snapcraft.yaml | 4 ++-- tests/letstest/scripts/test_sdists.sh | 4 ++-- ..._constraints.txt => certbot_requirements.txt} | 2 +- ...{dev_constraints.txt => dev_requirements.txt} | 4 ++-- tools/pip_install.py | 16 ++++++++-------- tools/pipstrap.py | 2 +- ...constraints.txt => pipstrap_requirements.txt} | 2 +- tools/rebuild_certbot_constraints.py | 8 ++++---- tools/snap/generate_dnsplugins_all.sh | 6 +++--- windows-installer/construct.py | 4 ++-- 10 files changed, 26 insertions(+), 26 deletions(-) rename tools/{certbot_constraints.txt => certbot_requirements.txt} (99%) rename tools/{dev_constraints.txt => dev_requirements.txt} (96%) rename tools/{pipstrap_constraints.txt => pipstrap_requirements.txt} (96%) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index d53fba88bd4..a8bf27615d1 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -85,8 +85,8 @@ parts: snapcraftctl build override-pull: | snapcraftctl pull - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/certbot_constraints.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_constraints.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/certbot_requirements.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_requirements.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" echo "$(python3 "${SNAPCRAFT_PART_SRC}/tools/merge_requirements.py" "${SNAPCRAFT_PART_SRC}/snap-constraints.txt")" > "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"` shared-metadata: diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index becdd6d9a3e..31f4ef72b8f 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -12,8 +12,8 @@ sudo $BOOTSTRAP_SCRIPT # We strip the hashes because the venv creation script includes unhashed # constraints in the commands given to pip and the mix of hashed and unhashed # packages makes pip error out. -python3 tools/strip_hashes.py tools/pipstrap_constraints.txt > constraints.txt -python3 tools/strip_hashes.py tools/certbot_constraints.txt > requirements.txt +python3 tools/strip_hashes.py tools/pipstrap_requirements.txt > constraints.txt +python3 tools/strip_hashes.py tools/certbot_requirements.txt > requirements.txt # We pin cryptography to 3.1.1 and pyOpenSSL to 19.1.0 specifically for CentOS 7 / RHEL 7 # because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been diff --git a/tools/certbot_constraints.txt b/tools/certbot_requirements.txt similarity index 99% rename from tools/certbot_constraints.txt rename to tools/certbot_requirements.txt index 77bfef9dbca..9ea882223c0 100644 --- a/tools/certbot_constraints.txt +++ b/tools/certbot_requirements.txt @@ -2,7 +2,7 @@ # To generate this, do (with docker and package hashin installed): # ``` # tools/rebuild_certbot_contraints.py \ -# tools/certbot_constraints.txt +# tools/certbot_requirements.txt # ``` # If you want to update a single dependency, run commands similar to these: # ``` diff --git a/tools/dev_constraints.txt b/tools/dev_requirements.txt similarity index 96% rename from tools/dev_constraints.txt rename to tools/dev_requirements.txt index f4059a3f90f..d4ba3789bb0 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_requirements.txt @@ -1,7 +1,7 @@ # Specifies Python package versions for development and building Docker images. # It includes in particular packages not specified in letsencrypt-auto's requirements file. -# Some dev package versions specified here may be overridden by higher level constraints -# files during tests (eg. tools/certbot_constraints.txt). +# Some dev package versions specified here may be overridden by higher level +# requirements files during tests (eg. tools/certbot_requirements.txt). alabaster==0.7.10 apacheconfig==0.3.2 apipkg==1.4 diff --git a/tools/pip_install.py b/tools/pip_install.py index e06650ff284..284aa7c0abf 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -1,10 +1,10 @@ #!/usr/bin/env python # pip installs packages using pinned package versions. If CERTBOT_OLDEST is set # to 1, a combination of tools/oldest_constraints.txt, -# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the +# tools/dev_requirements.txt, and local-oldest-requirements.txt contained in the # top level of the package's directory is used, otherwise, a combination of -# certbot-auto's requirements file and tools/dev_constraints.txt is used. The -# other file always takes precedence over tools/dev_constraints.txt. If +# certbot-auto's requirements file and tools/dev_requirements.txt is used. The +# other file always takes precedence over tools/dev_requirements.txt. If # CERTBOT_OLDEST is set, this script must be run with `-e ` and # no other arguments. @@ -57,10 +57,10 @@ def certbot_oldest_processing(tools_path, args, test_constraints): def certbot_normal_processing(tools_path, test_constraints): repo_path = os.path.dirname(tools_path) certbot_requirements = os.path.normpath(os.path.join( - repo_path, 'tools/certbot_constraints.txt')) + repo_path, 'tools/certbot_requirements.txt')) with open(certbot_requirements, 'r') as fd: certbot_reqs = fd.readlines() - with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd: + with open(os.path.join(tools_path, 'pipstrap_requirements.txt'), 'r') as fd: pipstrap_reqs = fd.readlines() with open(test_constraints, 'w') as fd: data_certbot = "\n".join(strip_hashes.process_entries(certbot_reqs)) @@ -74,11 +74,11 @@ def merge_requirements(tools_path, requirements, test_constraints, all_constrain # Indeed version retained for a given package will be the last version # found when following all requirements in the given order. # Here is the order by increasing priority: - # 1) The general development constraints (tools/dev_constraints.txt) + # 1) The general development constraints (tools/dev_requirements.txt) # 2) The general tests constraints (oldest_requirements.txt or - # certbot_constraints.txt + pipstrap's constraints for the normal processing) + # certbot_requirements.txt + pipstrap's constraints for the normal processing) # 3) The local requirement file, typically local-oldest-requirement in oldest tests - files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] + files = [os.path.join(tools_path, 'dev_requirements.txt'), test_constraints] if requirements: files.append(requirements) merged_requirements = merge_module.main(*files) diff --git a/tools/pipstrap.py b/tools/pipstrap.py index 2b2e3dcbbe9..c0516d9ed77 100755 --- a/tools/pipstrap.py +++ b/tools/pipstrap.py @@ -4,7 +4,7 @@ import pip_install -_REQUIREMENTS_PATH = os.path.join(os.path.dirname(__file__), "pipstrap_constraints.txt") +_REQUIREMENTS_PATH = os.path.join(os.path.dirname(__file__), "pipstrap_requirements.txt") def main(): diff --git a/tools/pipstrap_constraints.txt b/tools/pipstrap_requirements.txt similarity index 96% rename from tools/pipstrap_constraints.txt rename to tools/pipstrap_requirements.txt index 5de9e147df5..21bf9c0b1a3 100644 --- a/tools/pipstrap_constraints.txt +++ b/tools/pipstrap_requirements.txt @@ -1,4 +1,4 @@ -# Constraints for pipstrap.py +# Requirements file for pipstrap.py # # We include the hashes of the packages here for extra verification of # the packages downloaded from PyPI. This is especially valuable in our diff --git a/tools/rebuild_certbot_constraints.py b/tools/rebuild_certbot_constraints.py index f5e5d3ca78b..39571b2b17d 100755 --- a/tools/rebuild_certbot_constraints.py +++ b/tools/rebuild_certbot_constraints.py @@ -4,12 +4,12 @@ on various Linux distributions. It generates a requirements file contained the pinned and hashed versions, ready to be used by pip to install the certbot dependencies. -This script is typically used to update the certbot_constraints.txt file. +This script is typically used to update the certbot_requirements.txt file. To achieve its purpose, this script will start a certbot installation with unpinned dependencies, then gather them, on various distributions started as Docker containers. -Usage: tools/rebuild_certbot_constraints.py new_requirements.txt +Usage: tools/rebuild_certbot_requirements.py new_requirements.txt NB1: Docker must be installed on the machine running this script. NB2: Python library 'hashin' must be installed on the machine running this script. @@ -42,7 +42,7 @@ 'cryptography': '3.3.2', } -# ./certbot/tools/rebuild_certbot_constraints.py (2 levels from certbot root path) +# ./certbot/tools/rebuild_certbot_requirements.py (2 levels from certbot root path) CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__))) # The script will be used to gather dependencies for a given distribution. @@ -209,7 +209,7 @@ def _write_requirements(dest_file, requirements, conflicts): # To generate this, do (with docker and package hashin installed): # ``` # tools/rebuild_certbot_contraints.py \\ -# tools/certbot_constraints.txt +# tools/certbot_requirements.txt # ``` # If you want to update a single dependency, run commands similar to these: # ``` diff --git a/tools/snap/generate_dnsplugins_all.sh b/tools/snap/generate_dnsplugins_all.sh index 976b0dd7b3e..fdb80e6f60a 100755 --- a/tools/snap/generate_dnsplugins_all.sh +++ b/tools/snap/generate_dnsplugins_all.sh @@ -9,8 +9,8 @@ for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_snapcraft.sh $PLUGIN_PATH bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_postrefreshhook.sh $PLUGIN_PATH # Create constraints file - "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_constraints.txt \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/certbot_constraints.txt) \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_constraints.txt) \ + "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_requirements.txt \ + <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/certbot_requirements.txt) \ + <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_requirements.txt) \ > "${PLUGIN_PATH}"/snap-constraints.txt done diff --git a/windows-installer/construct.py b/windows-installer/construct.py index eb199a7e10a..a8bac904ea4 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -79,8 +79,8 @@ def _prepare_build_tools(venv_path, venv_python, repo_path): @contextlib.contextmanager def _prepare_constraints(repo_path): - reqs_certbot = os.path.join(repo_path, 'tools', 'certbot_constraints.txt') - reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_constraints.txt') + reqs_certbot = os.path.join(repo_path, 'tools', 'certbot_requirements.txt') + reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_requirements.txt') constraints_certbot = subprocess.check_output( [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_certbot], universal_newlines=True)