Dependency scanning (#6)

* certbot constraints to reqs

* rename dev constraints to reqs

* Rename pipstrap constraints to requirements

snap/snapcraft.yaml

@@ -85,8 +85,8 @@ parts:
       snapcraftctl build
     override-pull: |
       snapcraftctl pull
-      python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/certbot_constraints.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt"
-      python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_constraints.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt"
+      python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/certbot_requirements.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt"
+      python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_requirements.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt"
       echo "$(python3 "${SNAPCRAFT_PART_SRC}/tools/merge_requirements.py" "${SNAPCRAFT_PART_SRC}/snap-constraints.txt")" > "${SNAPCRAFT_PART_SRC}/snap-constraints.txt"
       snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"`
   shared-metadata:

tests/letstest/scripts/test_sdists.sh

@@ -12,8 +12,8 @@ sudo $BOOTSTRAP_SCRIPT
 # We strip the hashes because the venv creation script includes unhashed
 # constraints in the commands given to pip and the mix of hashed and unhashed
 # packages makes pip error out.
-python3 tools/strip_hashes.py tools/pipstrap_constraints.txt > constraints.txt
-python3 tools/strip_hashes.py tools/certbot_constraints.txt > requirements.txt
+python3 tools/strip_hashes.py tools/pipstrap_requirements.txt > constraints.txt
+python3 tools/strip_hashes.py tools/certbot_requirements.txt > requirements.txt
 
 # We pin cryptography to 3.1.1 and pyOpenSSL to 19.1.0 specifically for CentOS 7 / RHEL 7
 # because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been

tools/certbot_constraints.txt → tools/certbot_requirements.txt

@@ -2,7 +2,7 @@
 # To generate this, do (with docker and package hashin installed):
 # ```
 # tools/rebuild_certbot_contraints.py \
-#   tools/certbot_constraints.txt
+#   tools/certbot_requirements.txt
 # ```
 # If you want to update a single dependency, run commands similar to these:
 # ```

tools/dev_constraints.txt → tools/dev_requirements.txt

@@ -1,7 +1,7 @@
 # Specifies Python package versions for development and building Docker images.
 # It includes in particular packages not specified in letsencrypt-auto's requirements file.
-# Some dev package versions specified here may be overridden by higher level constraints
-# files during tests (eg. tools/certbot_constraints.txt).
+# Some dev package versions specified here may be overridden by higher level
+# requirements files during tests (eg. tools/certbot_requirements.txt).
 alabaster==0.7.10
 apacheconfig==0.3.2
 apipkg==1.4

tools/pip_install.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python
 # pip installs packages using pinned package versions. If CERTBOT_OLDEST is set
 # to 1, a combination of tools/oldest_constraints.txt,
-# tools/dev_constraints.txt, and local-oldest-requirements.txt contained in the
+# tools/dev_requirements.txt, and local-oldest-requirements.txt contained in the
 # top level of the package's directory is used, otherwise, a combination of
-# certbot-auto's requirements file and tools/dev_constraints.txt is used. The
-# other file always takes precedence over tools/dev_constraints.txt. If
+# certbot-auto's requirements file and tools/dev_requirements.txt is used. The
+# other file always takes precedence over tools/dev_requirements.txt. If
 # CERTBOT_OLDEST is set, this script must be run with `-e <package-name>` and
 # no other arguments.
 
@@ -57,10 +57,10 @@ def certbot_oldest_processing(tools_path, args, test_constraints):
 def certbot_normal_processing(tools_path, test_constraints):
     repo_path = os.path.dirname(tools_path)
     certbot_requirements = os.path.normpath(os.path.join(
-        repo_path, 'tools/certbot_constraints.txt'))
+        repo_path, 'tools/certbot_requirements.txt'))
     with open(certbot_requirements, 'r') as fd:
         certbot_reqs = fd.readlines()
-    with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd:
+    with open(os.path.join(tools_path, 'pipstrap_requirements.txt'), 'r') as fd:
         pipstrap_reqs = fd.readlines()
     with open(test_constraints, 'w') as fd:
         data_certbot = "\n".join(strip_hashes.process_entries(certbot_reqs))
@@ -74,11 +74,11 @@ def merge_requirements(tools_path, requirements, test_constraints, all_constrain
     # Indeed version retained for a given package will be the last version
     # found when following all requirements in the given order.
     # Here is the order by increasing priority:
-    # 1) The general development constraints (tools/dev_constraints.txt)
+    # 1) The general development constraints (tools/dev_requirements.txt)
     # 2) The general tests constraints (oldest_requirements.txt or
-    #    certbot_constraints.txt + pipstrap's constraints for the normal processing)
+    #    certbot_requirements.txt + pipstrap's constraints for the normal processing)
     # 3) The local requirement file, typically local-oldest-requirement in oldest tests
-    files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints]
+    files = [os.path.join(tools_path, 'dev_requirements.txt'), test_constraints]
     if requirements:
         files.append(requirements)
     merged_requirements = merge_module.main(*files)

tools/pipstrap.py

@@ -4,7 +4,7 @@
 
 import pip_install
 
-_REQUIREMENTS_PATH = os.path.join(os.path.dirname(__file__), "pipstrap_constraints.txt")
+_REQUIREMENTS_PATH = os.path.join(os.path.dirname(__file__), "pipstrap_requirements.txt")
 
 
 def main():

tools/pipstrap_constraints.txt → tools/pipstrap_requirements.txt

@@ -1,4 +1,4 @@
-# Constraints for pipstrap.py
+# Requirements file for pipstrap.py
 #
 # We include the hashes of the packages here for extra verification of
 # the packages downloaded from PyPI. This is especially valuable in our

tools/rebuild_certbot_constraints.py

@@ -4,12 +4,12 @@
 on various Linux distributions. It generates a requirements file contained the pinned and hashed
 versions, ready to be used by pip to install the certbot dependencies.
 
-This script is typically used to update the certbot_constraints.txt file.
+This script is typically used to update the certbot_requirements.txt file.
 
 To achieve its purpose, this script will start a certbot installation with unpinned dependencies,
 then gather them, on various distributions started as Docker containers.
 
-Usage: tools/rebuild_certbot_constraints.py new_requirements.txt
+Usage: tools/rebuild_certbot_requirements.py new_requirements.txt
 
 NB1: Docker must be installed on the machine running this script.
 NB2: Python library 'hashin' must be installed on the machine running this script.
@@ -42,7 +42,7 @@
     'cryptography': '3.3.2',
 }
 
-# ./certbot/tools/rebuild_certbot_constraints.py (2 levels from certbot root path)
+# ./certbot/tools/rebuild_certbot_requirements.py (2 levels from certbot root path)
 CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__)))
 
 # The script will be used to gather dependencies for a given distribution.
@@ -209,7 +209,7 @@ def _write_requirements(dest_file, requirements, conflicts):
 # To generate this, do (with docker and package hashin installed):
 # ```
 # tools/rebuild_certbot_contraints.py \\
-#   tools/certbot_constraints.txt
+#   tools/certbot_requirements.txt
 # ```
 # If you want to update a single dependency, run commands similar to these:
 # ```

tools/snap/generate_dnsplugins_all.sh

@@ -9,8 +9,8 @@ for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do
   bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_snapcraft.sh $PLUGIN_PATH
   bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_postrefreshhook.sh $PLUGIN_PATH
   # Create constraints file
-  "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_constraints.txt \
-    <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/certbot_constraints.txt) \
-    <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_constraints.txt) \
+  "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_requirements.txt \
+    <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/certbot_requirements.txt) \
+    <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_requirements.txt) \
     > "${PLUGIN_PATH}"/snap-constraints.txt
 done

windows-installer/construct.py

@@ -79,8 +79,8 @@ def _prepare_build_tools(venv_path, venv_python, repo_path):
 
 @contextlib.contextmanager
 def _prepare_constraints(repo_path):
-    reqs_certbot = os.path.join(repo_path, 'tools', 'certbot_constraints.txt')
-    reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_constraints.txt')
+    reqs_certbot = os.path.join(repo_path, 'tools', 'certbot_requirements.txt')
+    reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_requirements.txt')
     constraints_certbot = subprocess.check_output(
         [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_certbot],
         universal_newlines=True)