From f4fc3e636d295776f0e6be53763f401e1cf4c41e Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 25 Mar 2021 23:51:59 -0700 Subject: [PATCH] Redo the majority of Certbot's pinning system (#8741) * add initial pyproject.toml * add extra dependencies * add simple bash script * polish * reuse pipstrap * add requirements.txt * temporarily remove hashin dep * Switch to requirements.txt * remove hashin check * update requirements.txt again * remove unnecessary merge * pin back augeas * unpin cryptography * simplify pywin32 pinning * update comment * pin back pytest and pylint * pin back pytest-forked * pin back coverage * update script comments * fix pyopenssl case * add minimum poetry version * run pin.sh --- .gitignore | 1 + certbot/setup.py | 5 +- snap/snapcraft.yaml | 4 +- tests/letstest/scripts/test_sdists.sh | 13 +- tools/certbot_constraints.txt | 260 ------------------------ tools/dev_constraints.txt | 2 +- tools/pinning/pin.sh | 51 +++++ tools/pinning/pyproject.toml | 56 ++++++ tools/pip_install.py | 62 ++---- tools/rebuild_certbot_constraints.py | 277 -------------------------- tools/requirements.txt | 176 ++++++++++++++++ tools/snap/generate_dnsplugins_all.sh | 5 +- windows-installer/construct.py | 35 +--- 13 files changed, 323 insertions(+), 624 deletions(-) delete mode 100644 tools/certbot_constraints.txt create mode 100755 tools/pinning/pin.sh create mode 100644 tools/pinning/pyproject.toml delete mode 100755 tools/rebuild_certbot_constraints.py create mode 100644 tools/requirements.txt diff --git a/.gitignore b/.gitignore index 5169defd6a0..34b3fc99e7d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ dist*/ letsencrypt.log certbot.log letsencrypt-auto-source/letsencrypt-auto.sig.lzma.base64 +poetry.lock # coverage .coverage diff --git a/certbot/setup.py b/certbot/setup.py index 8b2874f20df..0f54179a30e 100644 --- a/certbot/setup.py +++ b/certbot/setup.py @@ -59,7 +59,7 @@ def read_file(filename, encoding='utf8'): # However environment markers are supported only with setuptools >= 36.2. # So this dependency is not added for old Linux distributions with old setuptools, # in order to allow these systems to build certbot from sources. -pywin32_req = 'pywin32>=300' # do not forget to edit pywin32 dependency accordingly in windows-installer/construct.py +pywin32_req = 'pywin32>=300' setuptools_known_environment_markers = (LooseVersion(setuptools_version) >= LooseVersion('36.2')) if setuptools_known_environment_markers: install_requires.append(pywin32_req + " ; sys_platform == 'win32'") @@ -79,6 +79,9 @@ def read_file(filename, encoding='utf8'): 'ipdb', 'mypy', 'PyGithub', + # 1.1.0+ is required for poetry to use the poetry-core library for the + # build system declared in tools/pinning/pyproject.toml. + 'poetry>=1.1.0', 'pylint', 'pytest', 'pytest-cov', diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index ccaafd93b14..b3f1349e07f 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -93,9 +93,7 @@ parts: snapcraftctl build override-pull: | snapcraftctl pull - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/certbot_constraints.txt" | grep -v python-augeas >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" - python3 "${SNAPCRAFT_PART_SRC}/tools/strip_hashes.py" "${SNAPCRAFT_PART_SRC}/tools/pipstrap_constraints.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" - echo "$(python3 "${SNAPCRAFT_PART_SRC}/tools/merge_requirements.py" "${SNAPCRAFT_PART_SRC}/snap-constraints.txt")" > "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" + grep -v python-augeas "${SNAPCRAFT_PART_SRC}/tools/requirements.txt" >> "${SNAPCRAFT_PART_SRC}/snap-constraints.txt" snapcraftctl set-version `grep -oP "__version__ = '\K.*(?=')" "${SNAPCRAFT_PART_SRC}/certbot/certbot/__init__.py"` shared-metadata: plugin: dump diff --git a/tests/letstest/scripts/test_sdists.sh b/tests/letstest/scripts/test_sdists.sh index ad729e0ebe4..aa385ad2f1b 100755 --- a/tests/letstest/scripts/test_sdists.sh +++ b/tests/letstest/scripts/test_sdists.sh @@ -18,18 +18,15 @@ tools/pip_install.py pytest # setup constraints TEMP_DIR=$(mktemp -d) CONSTRAINTS="$TEMP_DIR/constraints.txt" -# We strip the hashes because we don't have hashes of our local packages and -# the mix of hashed and unhashed packages makes pip error out. -python3 tools/strip_hashes.py tools/certbot_constraints.txt > "$CONSTRAINTS" -python3 tools/strip_hashes.py tools/pipstrap_constraints.txt >> "$CONSTRAINTS" +cp tools/requirements.txt "$CONSTRAINTS" -# We pin cryptography to 3.1.1 and pyOpenSSL to 19.1.0 specifically for CentOS 7 / RHEL 7 +# We pin cryptography to 3.1.1 and pyopenssl to 19.1.0 specifically for CentOS 7 / RHEL 7 # because these systems ship only with OpenSSL 1.0.2, and this OpenSSL version support has been -# dropped on cryptography>=3.2 and pyOpenSSL>=20.0.0. -# Using this old version of OpenSSL would break the cryptography and pyOpenSSL wheels builds. +# dropped on cryptography>=3.2 and pyopenssl>=20.0.0. +# Using this old version of OpenSSL would break the cryptography and pyopenssl wheels builds. if [ -f /etc/redhat-release ] && [ "$(. /etc/os-release 2> /dev/null && echo "$VERSION_ID" | cut -d '.' -f1)" -eq 7 ]; then sed -i 's|cryptography==.*|cryptography==3.1.1|g' "$CONSTRAINTS" - sed -i 's|pyOpenSSL==.*|pyOpenSSL==19.1.0|g' "$CONSTRAINTS" + sed -i 's|pyopenssl==.*|pyopenssl==19.1.0|g' "$CONSTRAINTS" fi diff --git a/tools/certbot_constraints.txt b/tools/certbot_constraints.txt deleted file mode 100644 index 2f48c8b3e62..00000000000 --- a/tools/certbot_constraints.txt +++ /dev/null @@ -1,260 +0,0 @@ -# This is the flattened list of pinned packages to build certbot deployable artifacts. -# To generate this, do (with docker and package hashin installed): -# ``` -# tools/rebuild_certbot_contraints.py \ -# tools/certbot_constraints.txt -# ``` -# If you want to update a single dependency, run commands similar to these: -# ``` -# pip install hashin -# hashin -r tools/certbot_constraints.txt cryptography==1.5.2 -# ``` -ConfigArgParse==1.4 \ - --hash=sha256:abef9ff44fb0091f0e3bb2ee7e5b26a02b5b62d45408a5272a9bd461f5b59b4b -certifi==2020.12.5 \ - --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ - --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 -cffi==1.14.5 \ - --hash=sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813 \ - --hash=sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06 \ - --hash=sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea \ - --hash=sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee \ - --hash=sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396 \ - --hash=sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73 \ - --hash=sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315 \ - --hash=sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1 \ - --hash=sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49 \ - --hash=sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892 \ - --hash=sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482 \ - --hash=sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058 \ - --hash=sha256:51182f8927c5af975fece87b1b369f722c570fe169f9880764b1ee3bca8347b5 \ - --hash=sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53 \ - --hash=sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045 \ - --hash=sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3 \ - --hash=sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5 \ - --hash=sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e \ - --hash=sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c \ - --hash=sha256:72d8d3ef52c208ee1c7b2e341f7d71c6fd3157138abf1a95166e6165dd5d4369 \ - --hash=sha256:8ae6299f6c68de06f136f1f9e69458eae58f1dacf10af5c17353eae03aa0d827 \ - --hash=sha256:8b198cec6c72df5289c05b05b8b0969819783f9418e0409865dac47288d2a053 \ - --hash=sha256:99cd03ae7988a93dd00bcd9d0b75e1f6c426063d6f03d2f90b89e29b25b82dfa \ - --hash=sha256:9cf8022fb8d07a97c178b02327b284521c7708d7c71a9c9c355c178ac4bbd3d4 \ - --hash=sha256:9de2e279153a443c656f2defd67769e6d1e4163952b3c622dcea5b08a6405322 \ - --hash=sha256:9e93e79c2551ff263400e1e4be085a1210e12073a31c2011dbbda14bda0c6132 \ - --hash=sha256:9ff227395193126d82e60319a673a037d5de84633f11279e336f9c0f189ecc62 \ - --hash=sha256:a465da611f6fa124963b91bf432d960a555563efe4ed1cc403ba5077b15370aa \ - --hash=sha256:ad17025d226ee5beec591b52800c11680fca3df50b8b29fe51d882576e039ee0 \ - --hash=sha256:afb29c1ba2e5a3736f1c301d9d0abe3ec8b86957d04ddfa9d7a6a42b9367e396 \ - --hash=sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e \ - --hash=sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991 \ - --hash=sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6 \ - --hash=sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1 \ - --hash=sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406 \ - --hash=sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d \ - --hash=sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c -chardet==4.0.0 \ - --hash=sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa \ - --hash=sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 -configobj==5.0.6 \ - --hash=sha256:a2f5650770e1c87fb335af19a9b7eb73fc05ccf22144eb68db7d00cd2bcb0902 -cryptography==3.4.6 \ - --hash=sha256:066bc53f052dfeda2f2d7c195cf16fb3e5ff13e1b6b7415b468514b40b381a5b \ - --hash=sha256:0923ba600d00718d63a3976f23cab19aef10c1765038945628cd9be047ad0336 \ - --hash=sha256:2d32223e5b0ee02943f32b19245b61a62db83a882f0e76cc564e1cec60d48f87 \ - --hash=sha256:4169a27b818de4a1860720108b55a2801f32b6ae79e7f99c00d79f2a2822eeb7 \ - --hash=sha256:57ad77d32917bc55299b16d3b996ffa42a1c73c6cfa829b14043c561288d2799 \ - --hash=sha256:5ecf2bcb34d17415e89b546dbb44e73080f747e504273e4d4987630493cded1b \ - --hash=sha256:600cf9bfe75e96d965509a4c0b2b183f74a4fa6f5331dcb40fb7b77b7c2484df \ - --hash=sha256:66b57a9ca4b3221d51b237094b0303843b914b7d5afd4349970bb26518e350b0 \ - --hash=sha256:93cfe5b7ff006de13e1e89830810ecbd014791b042cbe5eec253be11ac2b28f3 \ - --hash=sha256:9e98b452132963678e3ac6c73f7010fe53adf72209a32854d55690acac3f6724 \ - --hash=sha256:df186fcbf86dc1ce56305becb8434e4b6b7504bc724b71ad7a3239e0c9d14ef2 \ - --hash=sha256:fec7fb46b10da10d9e1d078d1ff8ed9e05ae14f431fdbd11145edd0550b9a964 -distro==1.5.0 \ - --hash=sha256:0e58756ae38fbd8fc3020d54badb8eae17c5b9dcbed388b17bb55b8a5928df92 \ - --hash=sha256:df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799 -idna==2.10 \ - --hash=sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6 \ - --hash=sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 -josepy==1.8.0 \ - --hash=sha256:6d632fcdaf0bed09e33f81f13b10575d4f0b7c37319350b725454e04a41e6a49 \ - --hash=sha256:a5a182eb499665d99e7ec54bb3fe389f9cbc483d429c9651f20384ba29564269 -parsedatetime==2.6 \ - --hash=sha256:4cb368fbb18a0b7231f4d76119165451c8d2e35951455dfee97c62a87b04d455 \ - --hash=sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b -pyOpenSSL==20.0.1 \ - --hash=sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51 \ - --hash=sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b -pyRFC3339==1.1 \ - --hash=sha256:67196cb83b470709c580bb4738b83165e67c6cc60e1f2e4f286cfcb402a926f4 \ - --hash=sha256:81b8cbe1519cdb79bed04910dd6fa4e181faf8c88dff1e1b987b5f7ab23a5b1a -pycparser==2.20 \ - --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \ - --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 -pyparsing==2.4.7 \ - --hash=sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1 \ - --hash=sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b -python-augeas==0.5.0 \ - --hash=sha256:67d59d66cdba8d624e0389b87b2a83a176f21f16a87553b50f5703b23f29bac2 -pytz==2021.1 \ - --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da \ - --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 -requests==2.25.1 \ - --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804 \ - --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e -requests-toolbelt==0.9.1 \ - --hash=sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f \ - --hash=sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0 -six==1.15.0 \ - --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \ - --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced -urllib3==1.26.4 \ - --hash=sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df \ - --hash=sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937 -zope.component==4.6.2 \ - --hash=sha256:607628e4c84f7887a69a958542b5c304663e726b73aba0882e3a3f059bff14f3 \ - --hash=sha256:91628918218b3e6f6323de2a7b845e09ddc5cae131c034896c051b084bba3c92 -zope.deferredimport==4.3.1 \ - --hash=sha256:57b2345e7b5eef47efcd4f634ff16c93e4265de3dcf325afc7315ade48d909e1 \ - --hash=sha256:9a0c211df44aa95f1c4e6d2626f90b400f56989180d3ef96032d708da3d23e0a -zope.deprecation==4.4.0 \ - --hash=sha256:0d453338f04bacf91bbfba545d8bcdf529aa829e67b705eac8c1a7fdce66e2df \ - --hash=sha256:f1480b74995958b24ce37b0ef04d3663d2683e5d6debc96726eff18acf4ea113 -zope.event==4.5.0 \ - --hash=sha256:2666401939cdaa5f4e0c08cf7f20c9b21423b95e88f4675b1443973bdb080c42 \ - --hash=sha256:5e76517f5b9b119acf37ca8819781db6c16ea433f7e2062c4afc2b6fbedb1330 -zope.hookable==5.0.1 \ - --hash=sha256:0194b9b9e7f614abba60c90b231908861036578297515d3d6508eb10190f266d \ - --hash=sha256:0c2977473918bdefc6fa8dfb311f154e7f13c6133957fe649704deca79b92093 \ - --hash=sha256:17b8bdb3b77e03a152ca0d5ca185a7ae0156f5e5a2dbddf538676633a1f7380f \ - --hash=sha256:29d07681a78042cdd15b268ae9decffed9ace68a53eebeb61d65ae931d158841 \ - --hash=sha256:36fb1b35d1150267cb0543a1ddd950c0bc2c75ed0e6e92e3aaa6ac2e29416cb7 \ - --hash=sha256:3aed60c2bb5e812bbf9295c70f25b17ac37c233f30447a96c67913ba5073642f \ - --hash=sha256:3cac1565cc768911e72ca9ec4ddf5c5109e1fef0104f19f06649cf1874943b60 \ - --hash=sha256:3d4bc0cc4a37c3cd3081063142eeb2125511db3c13f6dc932d899c512690378e \ - --hash=sha256:3f73096f27b8c28be53ffb6604f7b570fbbb82f273c6febe5f58119009b59898 \ - --hash=sha256:522d1153d93f2d48aa0bd9fb778d8d4500be2e4dcf86c3150768f0e3adbbc4ef \ - --hash=sha256:523d2928fb7377bbdbc9af9c0b14ad73e6eaf226349f105733bdae27efd15b5a \ - --hash=sha256:5848309d4fc5c02150a45e8f8d2227e5bfda386a508bbd3160fed7c633c5a2fa \ - --hash=sha256:6781f86e6d54a110980a76e761eb54590630fd2af2a17d7edf02a079d2646c1d \ - --hash=sha256:6fd27921ebf3aaa945fa25d790f1f2046204f24dba4946f82f5f0a442577c3e9 \ - --hash=sha256:70d581862863f6bf9e175e85c9d70c2d7155f53fb04dcdb2f73cf288ca559a53 \ - --hash=sha256:81867c23b0dc66c8366f351d00923f2bc5902820a24c2534dfd7bf01a5879963 \ - --hash=sha256:81db29edadcbb740cd2716c95a297893a546ed89db1bfe9110168732d7f0afdd \ - --hash=sha256:86bd12624068cea60860a0759af5e2c3adc89c12aef6f71cf12f577e28deefe3 \ - --hash=sha256:9c184d8f9f7a76e1ced99855ccf390ffdd0ec3765e5cbf7b9cada600accc0a1e \ - --hash=sha256:acc789e8c29c13555e43fe4bf9fcd15a65512c9645e97bbaa5602e3201252b02 \ - --hash=sha256:afaa740206b7660d4cc3b8f120426c85761f51379af7a5b05451f624ad12b0af \ - --hash=sha256:b5f5fa323f878bb16eae68ea1ba7f6c0419d4695d0248bed4b18f51d7ce5ab85 \ - --hash=sha256:bd89e0e2c67bf4ac3aca2a19702b1a37269fb1923827f68324ac2e7afd6e3406 \ - --hash=sha256:c212de743283ec0735db24ec6ad913758df3af1b7217550ff270038062afd6ae \ - --hash=sha256:ca553f524293a0bdea05e7f44c3e685e4b7b022cb37d87bc4a3efa0f86587a8d \ - --hash=sha256:cab67065a3db92f636128d3157cc5424a145f82d96fb47159c539132833a6d36 \ - --hash=sha256:d3b3b3eedfdbf6b02898216e85aa6baf50207f4378a2a6803d6d47650cd37031 \ - --hash=sha256:d9f4a5a72f40256b686d31c5c0b1fde503172307beb12c1568296e76118e402c \ - --hash=sha256:df5067d87aaa111ed5d050e1ee853ba284969497f91806efd42425f5348f1c06 \ - --hash=sha256:e2587644812c6138f05b8a41594a8337c6790e3baf9a01915e52438c13fc6bef \ - --hash=sha256:e27fd877662db94f897f3fd532ef211ca4901eb1a70ba456f15c0866a985464a \ - --hash=sha256:e427ebbdd223c72e06ba94c004bb04e996c84dec8a0fa84e837556ae145c439e \ - --hash=sha256:e583ad4309c203ef75a09d43434cf9c2b4fa247997ecb0dcad769982c39411c7 \ - --hash=sha256:e760b2bc8ece9200804f0c2b64d10147ecaf18455a2a90827fbec4c9d84f3ad5 \ - --hash=sha256:ea9a9cc8bcc70e18023f30fa2f53d11ae069572a162791224e60cd65df55fb69 \ - --hash=sha256:ecb3f17dce4803c1099bd21742cd126b59817a4e76a6544d31d2cca6e30dbffd \ - --hash=sha256:ed794e3b3de42486d30444fb60b5561e724ee8a2d1b17b0c2e0f81e3ddaf7a87 \ - --hash=sha256:ee885d347279e38226d0a437b6a932f207f691c502ee565aba27a7022f1285df \ - --hash=sha256:fd5e7bc5f24f7e3d490698f7b854659a9851da2187414617cd5ed360af7efd63 \ - --hash=sha256:fe45f6870f7588ac7b2763ff1ce98cce59369717afe70cc353ec5218bc854bcc -zope.interface==5.2.0 \ - --hash=sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1 \ - --hash=sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d \ - --hash=sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123 \ - --hash=sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232 \ - --hash=sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549 \ - --hash=sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102 \ - --hash=sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5 \ - --hash=sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45 \ - --hash=sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00 \ - --hash=sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc \ - --hash=sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7 \ - --hash=sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104 \ - --hash=sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034 \ - --hash=sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3 \ - --hash=sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3 \ - --hash=sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4 \ - --hash=sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86 \ - --hash=sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96 \ - --hash=sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546 \ - --hash=sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb \ - --hash=sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3 \ - --hash=sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b \ - --hash=sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b \ - --hash=sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec \ - --hash=sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae \ - --hash=sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e \ - --hash=sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386 \ - --hash=sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2 \ - --hash=sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a \ - --hash=sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d \ - --hash=sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a \ - --hash=sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24 \ - --hash=sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d \ - --hash=sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b \ - --hash=sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50 \ - --hash=sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523 \ - --hash=sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a \ - --hash=sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095 \ - --hash=sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a \ - --hash=sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520 \ - --hash=sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65 \ - --hash=sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11 \ - --hash=sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c \ - --hash=sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7 \ - --hash=sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332 \ - --hash=sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e \ - --hash=sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c \ - --hash=sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7 \ - --hash=sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20 \ - --hash=sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc \ - --hash=sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd \ - --hash=sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537 -zope.proxy==4.3.5 \ - --hash=sha256:00573dfa755d0703ab84bb23cb6ecf97bb683c34b340d4df76651f97b0bab068 \ - --hash=sha256:092049280f2848d2ba1b57b71fe04881762a220a97b65288bcb0968bb199ec30 \ - --hash=sha256:0cbd27b4d3718b5ec74fc65ffa53c78d34c65c6fd9411b8352d2a4f855220cf1 \ - --hash=sha256:17fc7e16d0c81f833a138818a30f366696653d521febc8e892858041c4d88785 \ - --hash=sha256:19577dfeb70e8a67249ba92c8ad20589a1a2d86a8d693647fa8385408a4c17b0 \ - --hash=sha256:207aa914576b1181597a1516e1b90599dc690c095343ae281b0772e44945e6a4 \ - --hash=sha256:219a7db5ed53e523eb4a4769f13105118b6d5b04ed169a283c9775af221e231f \ - --hash=sha256:2b50ea79849e46b5f4f2b0247a3687505d32d161eeb16a75f6f7e6cd81936e43 \ - --hash=sha256:5903d38362b6c716e66bbe470f190579c530a5baf03dbc8500e5c2357aa569a5 \ - --hash=sha256:5c24903675e271bd688c6e9e7df5775ac6b168feb87dbe0e4bcc90805f21b28f \ - --hash=sha256:5ef6bc5ed98139e084f4e91100f2b098a0cd3493d4e76f9d6b3f7b95d7ad0f06 \ - --hash=sha256:61b55ae3c23a126a788b33ffb18f37d6668e79a05e756588d9e4d4be7246ab1c \ - --hash=sha256:63ddb992931a5e616c87d3d89f5a58db086e617548005c7f9059fac68c03a5cc \ - --hash=sha256:6943da9c09870490dcfd50c4909c0cc19f434fa6948f61282dc9cb07bcf08160 \ - --hash=sha256:6ad40f85c1207803d581d5d75e9ea25327cd524925699a83dfc03bf8e4ba72b7 \ - --hash=sha256:6b44433a79bdd7af0e3337bd7bbcf53dd1f9b0fa66bf21bcb756060ce32a96c1 \ - --hash=sha256:6bbaa245015d933a4172395baad7874373f162955d73612f0b66b6c2c33b6366 \ - --hash=sha256:7007227f4ea85b40a2f5e5a244479f6a6dfcf906db9b55e812a814a8f0e2c28d \ - --hash=sha256:74884a0aec1f1609190ec8b34b5d58fb3b5353cf22b96161e13e0e835f13518f \ - --hash=sha256:7d25fe5571ddb16369054f54cdd883f23de9941476d97f2b92eb6d7d83afe22d \ - --hash=sha256:7e162bdc5e3baad26b2262240be7d2bab36991d85a6a556e48b9dfb402370261 \ - --hash=sha256:814d62678dc3a30f4aa081982d830b7c342cf230ffc9d030b020cb154eeebf9e \ - --hash=sha256:8878a34c5313ee52e20aa50b03138af8d472bae465710fb954d133a9bfd3c38d \ - --hash=sha256:a66a0d94e5b081d5d695e66d6667e91e74d79e273eee95c1747717ba9cb70792 \ - --hash=sha256:a69f5cbf4addcfdf03dda564a671040127a6b7c34cf9fe4973582e68441b63fa \ - --hash=sha256:b00f9f0c334d07709d3f73a7cb8ae63c6ca1a90c790a63b5e7effa666ef96021 \ - --hash=sha256:b6ed71e4a7b4690447b626f499d978aa13197a0e592950e5d7020308f6054698 \ - --hash=sha256:bdf5041e5851526e885af579d2f455348dba68d74f14a32781933569a327fddf \ - --hash=sha256:be034360dd34e62608419f86e799c97d389c10a0e677a25f236a971b2f40dac9 \ - --hash=sha256:cc8f590a5eed30b314ae6b0232d925519ade433f663de79cc3783e4b10d662ba \ - --hash=sha256:cd7a318a15fe6cc4584bf3c4426f092ed08c0fd012cf2a9173114234fe193e11 \ - --hash=sha256:cf19b5f63a59c20306e034e691402b02055c8f4e38bf6792c23cad489162a642 \ - --hash=sha256:cfc781ce442ec407c841e9aa51d0e1024f72b6ec34caa8fdb6ef9576d549acf2 \ - --hash=sha256:dea9f6f8633571e18bc20cad83603072e697103a567f4b0738d52dd0211b4527 \ - --hash=sha256:e4a86a1d5eb2cce83c5972b3930c7c1eac81ab3508464345e2b8e54f119d5505 \ - --hash=sha256:e7106374d4a74ed9ff00c46cc00f0a9f06a0775f8868e423f85d4464d2333679 \ - --hash=sha256:e98a8a585b5668aa9e34d10f7785abf9545fe72663b4bfc16c99a115185ae6a5 \ - --hash=sha256:f64840e68483316eb58d82c376ad3585ca995e69e33b230436de0cdddf7363f9 \ - --hash=sha256:f8f4b0a9e6683e43889852130595c8854d8ae237f2324a053cdd884de936aa9b \ - --hash=sha256:fc45a53219ed30a7f670a6d8c98527af0020e6fd4ee4c0a8fb59f147f06d816c diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index ae175cdd4f8..43a27a3d083 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,7 +1,7 @@ # Specifies Python package versions for development and building Docker images. # It includes in particular packages not specified in tools/certbot_constraints.txt. # Some dev package versions specified here may be overridden by higher level constraints -# files during tests (eg. tools/certbot_constraints.txt). +# files during tests (eg. tools/oldest_constraints.txt). alabaster==0.7.10 apacheconfig==0.3.2 apipkg==1.4 diff --git a/tools/pinning/pin.sh b/tools/pinning/pin.sh new file mode 100755 index 00000000000..ce47f02a0e0 --- /dev/null +++ b/tools/pinning/pin.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# This script accepts no arguments and automates the process of updating +# Certbot's dependencies. Dependencies can be pinned to older versions by +# modifying pyproject.toml in the same directory as this file. +set -euo pipefail + +WORK_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +REPO_ROOT="$(dirname "$(dirname "${WORK_DIR}")")" +PIPSTRAP_CONSTRAINTS="${REPO_ROOT}/tools/pipstrap_constraints.txt" +RELATIVE_SCRIPT_PATH="$(realpath --relative-to "$REPO_ROOT" "$WORK_DIR")/$(basename "${BASH_SOURCE[0]}")" +REQUIREMENTS_FILE="$REPO_ROOT/tools/requirements.txt" +STRIP_HASHES="${REPO_ROOT}/tools/strip_hashes.py" + +if ! command -v poetry >/dev/null; then + echo "Please install poetry." + echo "You may need to recreate Certbot's virtual environment and activate it." + exit 1 +fi + +cd "${WORK_DIR}" + +if [ -f poetry.lock ]; then + rm poetry.lock +fi + +poetry lock + +TEMP_REQUIREMENTS=$(mktemp) +trap 'rm poetry.lock; rm $TEMP_REQUIREMENTS' EXIT + +poetry export -o "${TEMP_REQUIREMENTS}" --without-hashes +# We need to remove local packages from the requirements file. +sed -i '/^acme @/d; /certbot/d;' "${TEMP_REQUIREMENTS}" +# Poetry currently will not include pip, setuptools, or wheel in lockfiles or +# requirements files. This was resolved by +# https://github.com/python-poetry/poetry/pull/2826, but as of writing this it +# hasn't been included in a release yet. For now, we continue to keep +# pipstrap's pinning separate which has the added benefit of having it continue +# to check hashes when pipstrap is run directly. +"${STRIP_HASHES}" "${PIPSTRAP_CONSTRAINTS}" >> "${TEMP_REQUIREMENTS}" + +cat << EOF > "$REQUIREMENTS_FILE" +# This file was generated by $RELATIVE_SCRIPT_PATH and can be updated using +# that script. +# +# It is normally used as constraints to pip, however, it has the name +# requirements.txt so that is scanned by GitHub. See +# https://docs.github.com/en/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems +# for more info. +EOF +cat "${TEMP_REQUIREMENTS}" >> "${REQUIREMENTS_FILE}" diff --git a/tools/pinning/pyproject.toml b/tools/pinning/pyproject.toml new file mode 100644 index 00000000000..553ef947653 --- /dev/null +++ b/tools/pinning/pyproject.toml @@ -0,0 +1,56 @@ +[tool.poetry] +name = "certbot-pinner" +version = "0.1.0" +description = "A simple project for pinning Certbot's dependencies using Poetry." +authors = ["Certbot Project"] +license = "Apache License 2.0" + +[tool.poetry.dependencies] +python = "^3.6" + +# Local dependencies +# Any local packages that have dependencies on other local packages must be +# listed below before the package it depends on. For instance, certbot depends +# on acme so certbot must be listed before acme. +certbot-ci = {path = "../../certbot-ci", extras = ["docs"]} +certbot-compatibility-test = {path = "../../certbot-compatibility-test", extras = ["docs"]} +certbot-dns-cloudflare = {path = "../../certbot-dns-cloudflare", extras = ["docs"]} +certbot-dns-cloudxns = {path = "../../certbot-dns-cloudxns", extras = ["docs"]} +certbot-dns-digitalocean = {path = "../../certbot-dns-digitalocean", extras = ["docs"]} +certbot-dns-dnsimple = {path = "../../certbot-dns-dnsimple", extras = ["docs"]} +certbot-dns-dnsmadeeasy = {path = "../../certbot-dns-dnsmadeeasy", extras = ["docs"]} +certbot-dns-gehirn = {path = "../../certbot-dns-gehirn", extras = ["docs"]} +certbot-dns-google = {path = "../../certbot-dns-google", extras = ["docs"]} +certbot-dns-linode = {path = "../../certbot-dns-linode", extras = ["docs"]} +certbot-dns-luadns = {path = "../../certbot-dns-luadns", extras = ["docs"]} +certbot-dns-nsone = {path = "../../certbot-dns-nsone", extras = ["docs"]} +certbot-dns-ovh = {path = "../../certbot-dns-ovh", extras = ["docs"]} +certbot-dns-rfc2136 = {path = "../../certbot-dns-rfc2136", extras = ["docs"]} +certbot-dns-route53 = {path = "../../certbot-dns-route53", extras = ["docs"]} +certbot-dns-sakuracloud = {path = "../../certbot-dns-sakuracloud", extras = ["docs"]} +certbot-nginx = {path = "../../certbot-nginx", extras = ["docs"]} +certbot-apache = {path = "../../certbot-apache", extras = ["dev"]} +certbot = {path = "../../certbot", extras = ["dev", "docs"]} +acme = {path = "../../acme", extras = ["dev", "docs"]} + +# Extra dependencies +# See https://github.com/certbot/certbot/issues/8425. +mypy = "0.710" +# Upgrading coverage, pylint, pytest, and some of pytest's plugins causes many +# test failures so let's pin these packages back for now. +coverage = "4.5.4" +pylint = "2.4.3" +pytest = "3.2.5" +pytest-forked = "0.2" +# We were originally pinning back python-augeas for certbot-auto because we +# found the way older versions of the library linked to Augeas were more +# reliable. That's no longer a concern, however, we continue to pin back the +# library for now because it causes Certbot tests on Windows to fail. See +# https://github.com/certbot/certbot/issues/8732. +python-augeas = "0.5.0" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/tools/pip_install.py b/tools/pip_install.py index 4396af17a39..054ca1229b0 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -39,51 +39,34 @@ def find_tools_path(): return os.path.dirname(readlink.main(__file__)) -def certbot_oldest_processing(tools_path, args, test_constraints): +def certbot_oldest_processing(tools_path, args, constraints_path): if args[0] != '-e' or len(args) != 2: raise ValueError('When CERTBOT_OLDEST is set, this script must be run ' 'with a single -e argument.') # remove any extras such as [dev] pkg_dir = re.sub(r'\[\w+\]', '', args[1]) + # The order of the files in this list matters as files specified later can + # override the pinnings found in earlier files. + pinning_files = [os.path.join(tools_path, 'dev_constraints.txt'), + os.path.join(tools_path, 'oldest_constraints.txt')] requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') - shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) # packages like acme don't have any local oldest requirements - if not os.path.isfile(requirements): - return None - + if os.path.isfile(requirements): + # We add requirements to the end of the list so it can override + # anything that it needs to. + pinning_files.append(requirements) + else: + requirements = None + with open(constraints_path, 'w') as fd: + fd.write(merge_module.main(*pinning_files)) return requirements -def certbot_normal_processing(tools_path, test_constraints): +def certbot_normal_processing(tools_path, constraints_path): repo_path = os.path.dirname(tools_path) - certbot_requirements = os.path.normpath(os.path.join( - repo_path, 'tools/certbot_constraints.txt')) - with open(certbot_requirements, 'r') as fd: - certbot_reqs = fd.readlines() - with open(os.path.join(tools_path, 'pipstrap_constraints.txt'), 'r') as fd: - pipstrap_reqs = fd.readlines() - with open(test_constraints, 'w') as fd: - data_certbot = "\n".join(strip_hashes.process_entries(certbot_reqs)) - data_pipstrap = "\n".join(strip_hashes.process_entries(pipstrap_reqs)) - data = "\n".join([data_certbot, data_pipstrap]) - fd.write(data) - - -def merge_requirements(tools_path, requirements, test_constraints, all_constraints): - # Order of the files in the merge function matters. - # Indeed version retained for a given package will be the last version - # found when following all requirements in the given order. - # Here is the order by increasing priority: - # 1) The general development constraints (tools/dev_constraints.txt) - # 2) The general tests constraints (oldest_requirements.txt or - # certbot_constraints.txt + pipstrap's constraints for the normal processing) - # 3) The local requirement file, typically local-oldest-requirement in oldest tests - files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] - if requirements: - files.append(requirements) - merged_requirements = merge_module.main(*files) - with open(all_constraints, 'w') as fd: - fd.write(merged_requirements) + requirements = os.path.normpath(os.path.join( + repo_path, 'tools/requirements.txt')) + shutil.copy(requirements, constraints_path) def call_with_print(command, env=None): @@ -104,24 +87,21 @@ def main(args): tools_path = find_tools_path() with temporary_directory() as working_dir: - test_constraints = os.path.join(working_dir, 'test_constraints.txt') - all_constraints = os.path.join(working_dir, 'all_constraints.txt') - if os.environ.get('CERTBOT_NO_PIN') == '1': # With unpinned dependencies, there is no constraint pip_install_with_print(' '.join(args)) else: # Otherwise, we merge requirements to build the constraints and pin dependencies + constraints_path = os.path.join(working_dir, 'constraints.txt') requirements = None if os.environ.get('CERTBOT_OLDEST') == '1': - requirements = certbot_oldest_processing(tools_path, args, test_constraints) + requirements = certbot_oldest_processing(tools_path, args, constraints_path) else: - certbot_normal_processing(tools_path, test_constraints) + certbot_normal_processing(tools_path, constraints_path) env = os.environ.copy() - env["PIP_CONSTRAINT"] = all_constraints + env["PIP_CONSTRAINT"] = constraints_path - merge_requirements(tools_path, requirements, test_constraints, all_constraints) if requirements: # This branch is executed during the oldest tests # First step, install the transitive dependencies of oldest requirements # in respect with oldest constraints. diff --git a/tools/rebuild_certbot_constraints.py b/tools/rebuild_certbot_constraints.py deleted file mode 100755 index 14479d8eceb..00000000000 --- a/tools/rebuild_certbot_constraints.py +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env python -""" -Gather and consolidate the up-to-date dependencies available and required to install certbot -on various Linux distributions. It generates a requirements file contained the pinned and hashed -versions, ready to be used by pip to install the certbot dependencies. - -This script is typically used to update the certbot_constraints.txt file. - -To achieve its purpose, this script will start a certbot installation with unpinned dependencies, -then gather them, on various distributions started as Docker containers. - -Usage: tools/rebuild_certbot_constraints.py new_requirements.txt - -NB1: Docker must be installed on the machine running this script. -NB2: Python library 'hashin' must be installed on the machine running this script. -""" -from __future__ import print_function -import re -import shutil -import subprocess -import tempfile -import os -from os.path import dirname, abspath, join -import sys -import argparse - -# The list of docker distributions to test dependencies against with. -DISTRIBUTION_LIST = [ - 'ubuntu:20.04', 'ubuntu:18.04', 'debian:buster', - 'centos:8', 'centos:7', 'fedora:29', -] - -# These constraints will be added while gathering dependencies on each distribution. -# It can be used because a particular version for a package is required for any reason, -# or to solve a version conflict between two distributions requirements. -AUTHORITATIVE_CONSTRAINTS = { - # Too touchy to move to a new version. And will be removed soon - # in favor of pure python parser for Apache. - 'python-augeas': '0.5.0', -} - -# ./certbot/tools/rebuild_certbot_constraints.py (2 levels from certbot root path) -CERTBOT_REPO_PATH = dirname(dirname(abspath(__file__))) - -# The script will be used to gather dependencies for a given distribution. -# - bootstrap_os_packages.sh is used to install relevant OS packages, and set up an initial venv -# - then this venv is used to consistently construct an empty new venv -# - once pipstrap.py, this new venv pip-installs certbot runtime (including apache/nginx), -# without pinned dependencies, and respecting input authoritative requirements -# - `certbot plugins` is called to check we have a healthy environment -# - finally current set of dependencies is extracted out of the docker using pip freeze -SCRIPT = r"""#!/bin/sh -set -ex - -cd /tmp/certbot -tests/letstest/scripts/bootstrap_os_packages.sh - -python3 -m venv /tmp/venv - -/tmp/venv/bin/python tools/pipstrap.py -/tmp/venv/bin/pip install -e acme -e certbot -e certbot-apache -e certbot-nginx -c /tmp/constraints.txt -/tmp/venv/bin/certbot plugins -/tmp/venv/bin/pip freeze >> /tmp/workspace/requirements.txt -""" - - -def _read_from(file): - """Read all content of the file, and return it as a string.""" - with open(file, 'r') as file_h: - return file_h.read() - - -def _write_to(file, content): - """Write given string content to the file, overwriting its initial content.""" - with open(file, 'w') as file_h: - file_h.write(content) - - -def _requirements_from_one_distribution(distribution, verbose): - """ - Calculate the Certbot dependencies expressed for the given distribution, using the official - Docker for this distribution, and return the lines of the generated requirements file. - """ - print('===> Gathering dependencies for {0}.'.format(distribution)) - workspace = tempfile.mkdtemp() - script = join(workspace, 'script.sh') - authoritative_constraints = join(workspace, 'constraints.txt') - cid_file = join(workspace, 'cid') - - try: - _write_to(script, SCRIPT) - os.chmod(script, 0o755) - - _write_to(authoritative_constraints, '\n'.join( - '{0}=={1}'.format(package, version) for package, version in AUTHORITATIVE_CONSTRAINTS.items())) - - command = ['docker', 'run', '--rm', '--cidfile', cid_file, - '--network=host', - '-v', '{0}:/tmp/certbot'.format(CERTBOT_REPO_PATH), - '-v', '{0}:/tmp/workspace'.format(workspace), - '-v', '{0}:/tmp/constraints.txt'.format(authoritative_constraints), - distribution, '/tmp/workspace/script.sh'] - sub_stdout = sys.stdout if verbose else subprocess.PIPE - sub_stderr = sys.stderr if verbose else subprocess.STDOUT - process = subprocess.Popen(command, stdout=sub_stdout, stderr=sub_stderr, universal_newlines=True) - stdoutdata, _ = process.communicate() - - if process.returncode: - if stdoutdata: - sys.stderr.write('Output was:\n{0}'.format(stdoutdata)) - raise RuntimeError('Error while gathering dependencies for {0}.'.format(distribution)) - - with open(join(workspace, 'requirements.txt'), 'r') as file_h: - return file_h.readlines() - finally: - if os.path.isfile(cid_file): - cid = _read_from(cid_file) - try: - subprocess.check_output(['docker', 'kill', cid], stderr=subprocess.PIPE) - except subprocess.CalledProcessError: - pass - shutil.rmtree(workspace) - - -def _parse_and_merge_requirements(dependencies_map, requirements_file_lines, distribution): - """ - Extract every requirement from the given requirements file, and merge it in the dependency map. - Merging here means that the map contain every encountered dependency, and the version used in - each distribution. - - Example: - # dependencies_map = { - # } - _parse_and_merge_requirements(['cryptography=='1.2','requests=='2.1.0'], dependencies_map, 'debian:stretch') - # dependencies_map = { - # 'cryptography': [('1.2', 'debian:stretch)], - # 'requests': [('2.1.0', 'debian:stretch')] - # } - _parse_and_merge_requirements(['requests=='2.4.0', 'mock==1.3'], dependencies_map, 'centos:7') - # dependencies_map = { - # 'cryptography': [('1.2', 'debian:stretch)], - # 'requests': [('2.1.0', 'debian:stretch'), ('2.4.0', 'centos:7')], - # 'mock': [('2.4.0', 'centos:7')] - # } - """ - for line in requirements_file_lines: - match = re.match(r'([^=]+)==([^=]+)', line.strip()) - if not line.startswith('-e') and not line.startswith('#') and match: - package, version = match.groups() - if package not in ['acme', 'certbot', 'certbot-apache', 'certbot-nginx', 'pkg-resources']: - dependencies_map.setdefault(package, []).append((version, distribution)) - - -def _consolidate_and_validate_dependencies(dependency_map): - """ - Given the dependency map of all requirements found in all distributions for Certbot, - construct an array containing the unit requirements for Certbot to be used by pip, - and the version conflicts, if any, between several distributions for a package. - Return requirements and conflicts as a tuple. - """ - print('===> Consolidate and validate the dependency map.') - requirements = [] - conflicts = [] - for package, versions in dependency_map.items(): - reduced_versions = _reduce_versions(versions) - - if len(reduced_versions) > 1: - version_list = ['{0} ({1})'.format(version, ','.join(distributions)) - for version, distributions in reduced_versions.items()] - conflict = ('package {0} is declared with several versions: {1}' - .format(package, ', '.join(version_list))) - conflicts.append(conflict) - sys.stderr.write('ERROR: {0}\n'.format(conflict)) - else: - requirements.append((package, list(reduced_versions)[0])) - - requirements.sort(key=lambda x: x[0]) - return requirements, conflicts - - -def _reduce_versions(version_dist_tuples): - """ - Get an array of version/distribution tuples, - and reduce it to a map based on the version values. - - Example: [('1.2.0', 'debian:stretch'), ('1.4.0', 'ubuntu:18.04'), ('1.2.0', 'centos:6')] - => {'1.2.0': ['debiqn:stretch', 'centos:6'], '1.4.0': ['ubuntu:18.04']} - """ - version_dist_map = {} - for version, distribution in version_dist_tuples: - version_dist_map.setdefault(version, []).append(distribution) - - return version_dist_map - - -def _write_requirements(dest_file, requirements, conflicts): - """ - Given the list of requirements and conflicts, write a well-formatted requirements file, - whose requirements are hashed signed using hashin library. Conflicts are written at the end - of the generated file. - """ - print('===> Calculating hashes for the requirement file.') - - _write_to(dest_file, '''\ -# This is the flattened list of pinned packages to build certbot deployable artifacts. -# To generate this, do (with docker and package hashin installed): -# ``` -# tools/rebuild_certbot_contraints.py \\ -# tools/certbot_constraints.txt -# ``` -# If you want to update a single dependency, run commands similar to these: -# ``` -# pip install hashin -# hashin -r tools/certbot_constraints.txt cryptography==1.5.2 -# ``` -''') - - for req in requirements: - if req[0] in AUTHORITATIVE_CONSTRAINTS: - # If requirement is in AUTHORITATIVE_CONSTRAINTS, take its value instead of the - # computed one to get any environment descriptor that would have been added. - req = (req[0], AUTHORITATIVE_CONSTRAINTS[req[0]]) - subprocess.check_call(['hashin', '{0}=={1}'.format(req[0], req[1]), - '--requirements-file', dest_file]) - - if conflicts: - with open(dest_file, 'a') as file_h: - file_h.write('\n## ! SOME ERRORS OCCURRED ! ##\n') - file_h.write('\n'.join('# {0}'.format(conflict) for conflict in conflicts)) - file_h.write('\n') - - return _read_from(dest_file) - - -def _gather_dependencies(dest_file, verbose): - """ - Main method of this script. Given a destination file path, will write the file - containing the consolidated and hashed requirements for Certbot, validated - against several Linux distributions. - """ - dependencies_map = {} - - for distribution in DISTRIBUTION_LIST: - requirements_file_lines = _requirements_from_one_distribution(distribution, verbose) - _parse_and_merge_requirements(dependencies_map, requirements_file_lines, distribution) - - requirements, conflicts = _consolidate_and_validate_dependencies(dependencies_map) - - return _write_requirements(dest_file, requirements, conflicts) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=('Build a sanitized, pinned and hashed requirements file for certbot deployable' - ' artifacts, validated against several OS distributions using Docker.')) - parser.add_argument('requirements_path', - help='path for the generated requirements file') - parser.add_argument('--verbose', '-v', action='store_true', - help='verbose will display all output during docker execution') - - namespace = parser.parse_args() - - try: - subprocess.check_output(['hashin', '--version']) - except subprocess.CalledProcessError: - raise RuntimeError('Python library hashin is not installed in the current environment.') - - try: - subprocess.check_output(['docker', '--version'], stderr=subprocess.STDOUT) - except subprocess.CalledProcessError: - raise RuntimeError('Docker is not installed or accessible to current user.') - - file_content = _gather_dependencies(namespace.requirements_path, namespace.verbose) - - print(file_content) - print('===> Rebuilt requirement file is available on path {0}' - .format(abspath(namespace.requirements_path))) diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 00000000000..89c57c97447 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,176 @@ +# This file was generated by tools/pinning/pin.sh and can be updated using +# that script. +# +# It is normally used as constraints to pip, however, it has the name +# requirements.txt so that is scanned by GitHub. See +# https://docs.github.com/en/github/visualizing-repository-data-with-graphs/about-the-dependency-graph#supported-package-ecosystems +# for more info. +alabaster==0.7.12; python_version >= "3.6" +apacheconfig==0.3.2; python_version >= "3.6" +apipkg==1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +appdirs==1.4.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +appnope==0.1.2 +astroid==2.3.3; python_version >= "3.6" +attrs==20.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +azure-devops==6.0.0b4; python_version >= "3.6" +babel==2.9.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +backcall==0.2.0 +bcrypt==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +beautifulsoup4==4.9.3; python_version >= "3.6" and python_version < "4.0" +bleach==3.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +boto3==1.17.37; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +botocore==1.20.37; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +cachecontrol==0.12.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +cached-property==1.5.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +cachetools==4.2.1; python_version >= "3.5" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +cachy==0.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +cffi==1.14.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +chardet==4.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +cleo==0.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +clikit==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +cloudflare==2.8.15; python_version >= "3.6" +colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +configargparse==1.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +configobj==5.0.6; python_version >= "3.6" +coverage==4.5.4; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.3.0" and python_version < "4") +crashtest==0.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") +cryptography==3.4.7; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" +decorator==4.4.2; python_version == "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.2.0" +deprecated==1.2.12; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +distlib==0.3.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +distro==1.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +dns-lexicon==3.5.5; python_version >= "3.6" and python_version < "4.0" +dnspython==2.1.0; python_version >= "3.6" +docker-compose==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docker==4.2.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +dockerpty==0.4.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docopt==0.6.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") +execnet==1.8.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +filelock==3.0.12; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "4.0" +future==0.18.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_version < "4.0" and python_full_version >= "3.3.0" +google-api-core==1.26.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +google-api-python-client==2.0.2; python_version >= "3.6" +google-auth-httplib2==0.1.0; python_version >= "3.6" +google-auth==1.28.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +googleapis-common-protos==1.53.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +html5lib==1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +httplib2==0.19.0; python_version >= "3.6" +idna==2.10; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.5.0" and python_version >= "3.6" and python_version < "4.0" +imagesize==1.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +importlib-metadata==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.8" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") +importlib-resources==5.1.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7" +ipdb==0.13.7; python_version >= "3.6" +ipython-genutils==0.2.0; python_version == "3.6" +ipython==7.16.1; python_version == "3.6" +ipython==7.21.0; python_version >= "3.7" +isodate==0.6.0; python_version >= "3.6" +isort==4.3.21; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +jedi==0.18.0 +jeepney==0.6.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" +jinja2==2.11.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +josepy==1.8.0; python_version >= "3.6" +jsonlines==2.0.0; python_version >= "3.6" +jsonpickle==2.0.0; python_version >= "3.6" +jsonschema==3.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +keyring==21.8.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") +lazy-object-proxy==1.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +lockfile==0.12.2 +markupsafe==1.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +mccabe==0.6.1; python_version >= "3.6" +msgpack==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +msrest==0.6.21; python_version >= "3.6" +mypy-extensions==0.4.3; python_version >= "3.6" +mypy==0.710 +oauth2client==4.1.3; python_version >= "3.6" +oauthlib==3.1.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +packaging==20.9; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" +paramiko==2.7.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +parsedatetime==2.6; python_version >= "3.6" +parso==0.8.1; python_version == "3.6" +pastel==0.2.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pexpect==4.8.0 +pickleshare==0.7.5 +pkginfo==1.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +ply==3.11; python_version >= "3.6" +poetry-core==1.0.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +poetry==1.1.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +prompt-toolkit==3.0.3 +protobuf==3.15.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +ptyprocess==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pyasn1-modules==0.2.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +pyasn1==0.4.8; python_version >= "3.6" and python_version < "4" +pycparser==2.20; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pygithub==1.54.1; python_version >= "3.6" +pygments==2.8.1 +pyjwt==1.7.1; python_version >= "3.6" +pylev==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +pylint==2.4.3; python_version >= "3.5" +pynacl==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pyopenssl==20.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pypiwin32==223; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") +pyrfc3339==1.1; python_version >= "3.6" +pyrsistent==0.17.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytest-cov==2.6.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytest-forked==0.2 +pytest-xdist==1.24.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytest==3.2.5 +python-augeas==0.5.0 +python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +python-digitalocean==1.16.0; python_version >= "3.6" +python-dotenv==0.15.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +pytz==2021.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.6.0" +pywin32-ctypes==0.2.0; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "win32" +pywin32==300; sys_platform == "win32" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6") +pyyaml==5.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_full_version >= "3.6.0" and python_version >= "3.6" and python_version < "4.0" +readme-renderer==29.0; python_version >= "3.6" +repoze.sphinx.autointerface==0.8; python_version >= "3.6" +requests-file==1.5.1; python_version >= "3.6" and python_version < "4.0" +requests-oauthlib==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +requests-toolbelt==0.9.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +requests==2.25.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.6" and python_full_version >= "3.6.0" and python_version < "4.0" +rfc3986==1.4.0; python_version >= "3.6" +rsa==4.7.2; python_version >= "3.6" and python_version < "4" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6") +s3transfer==0.3.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" +secretstorage==3.3.1; python_version >= "3.6" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0") and sys_platform == "linux" +shellingham==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +six==1.15.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +snowballstemmer==2.1.0; python_version >= "3.6" +soupsieve==2.2.1; python_version >= "3.6" +sphinx-rtd-theme==0.5.1; python_version >= "3.6" +sphinx==3.5.3; python_version >= "3.6" +sphinxcontrib-applehelp==1.0.2; python_version >= "3.6" +sphinxcontrib-devhelp==1.0.2; python_version >= "3.6" +sphinxcontrib-htmlhelp==1.0.3; python_version >= "3.6" +sphinxcontrib-jsmath==1.0.1; python_version >= "3.6" +sphinxcontrib-qthelp==1.0.3; python_version >= "3.6" +sphinxcontrib-serializinghtml==1.1.4; python_version >= "3.6" +texttable==1.6.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +tldextract==3.1.0; python_version >= "3.6" and python_version < "4.0" +toml==0.10.2; python_version == "3.6" and python_full_version < "3.0.0" or python_version > "3.6" and python_full_version < "3.0.0" or python_version == "3.6" and python_full_version >= "3.5.0" or python_version > "3.6" and python_full_version >= "3.5.0" +tomlkit==0.7.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +tox==3.23.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +tqdm==4.59.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0" +traitlets==4.3.3 +twine==3.3.0; python_version >= "3.6" +typed-ast==1.4.2; python_version >= "3.6" and implementation_name == "cpython" and python_version < "3.8" +uritemplate==3.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" +urllib3==1.26.4; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" +virtualenv==20.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +wcwidth==0.2.5; python_version == "3.6" +webencodings==0.5.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.5.0" +websocket-client==0.58.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +wrapt==1.11.2; python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_version >= "3.6" and python_full_version >= "3.4.0") +zipp==3.4.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_full_version >= "3.5.0" and python_version < "3.7" +zope.component==5.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.event==4.5.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.hookable==5.0.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +zope.interface==5.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" +pip==20.2.4 +setuptools==54.1.2 +wheel==0.35.1 diff --git a/tools/snap/generate_dnsplugins_all.sh b/tools/snap/generate_dnsplugins_all.sh index 976b0dd7b3e..769b4ef4dee 100755 --- a/tools/snap/generate_dnsplugins_all.sh +++ b/tools/snap/generate_dnsplugins_all.sh @@ -9,8 +9,5 @@ for PLUGIN_PATH in "${CERTBOT_DIR}"/certbot-dns-*; do bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_snapcraft.sh $PLUGIN_PATH bash "${CERTBOT_DIR}"/tools/snap/generate_dnsplugins_postrefreshhook.sh $PLUGIN_PATH # Create constraints file - "${CERTBOT_DIR}"/tools/merge_requirements.py tools/dev_constraints.txt \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/certbot_constraints.txt) \ - <("${CERTBOT_DIR}"/tools/strip_hashes.py tools/pipstrap_constraints.txt) \ - > "${PLUGIN_PATH}"/snap-constraints.txt + cp "${CERTBOT_DIR}"/tools/requirements.txt "${PLUGIN_PATH}"/snap-constraints.txt done diff --git a/windows-installer/construct.py b/windows-installer/construct.py index eb199a7e10a..21ee4c8be85 100644 --- a/windows-installer/construct.py +++ b/windows-installer/construct.py @@ -12,7 +12,6 @@ PYTHON_VERSION = (3, 8, 8) PYTHON_BITNESS = 32 -PYWIN32_VERSION = 300 # do not forget to edit pywin32 dependency accordingly in setup.py NSIS_VERSION = '3.06.1' @@ -46,12 +45,12 @@ def _compile_wheels(repo_path, build_path, venv_python): # certbot_packages.extend([name for name in os.listdir(repo_path) if name.startswith('certbot-dns-')]) wheels_project = [os.path.join(repo_path, package) for package in certbot_packages] - with _prepare_constraints(repo_path) as constraints_file_path: - env = os.environ.copy() - env['PIP_CONSTRAINT'] = constraints_file_path - command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path] - command.extend(wheels_project) - subprocess.check_call(command, env=env) + constraints_file_path = os.path.join(repo_path, 'tools', 'requirements.txt') + env = os.environ.copy() + env['PIP_CONSTRAINT'] = constraints_file_path + command = [venv_python, '-m', 'pip', 'wheel', '-w', wheels_path] + command.extend(wheels_project) + subprocess.check_call(command, env=env) # Cryptography uses now a unique wheel name "cryptography-VERSION-cpXX-abi3-win32.whl where # cpXX is the lowest supported version of Python (eg. cp36 says that the wheel is compatible @@ -77,28 +76,6 @@ def _prepare_build_tools(venv_path, venv_python, repo_path): subprocess.check_call(['choco', 'upgrade', '--allow-downgrade', '-y', 'nsis', '--version', NSIS_VERSION]) -@contextlib.contextmanager -def _prepare_constraints(repo_path): - reqs_certbot = os.path.join(repo_path, 'tools', 'certbot_constraints.txt') - reqs_pipstrap = os.path.join(repo_path, 'tools', 'pipstrap_constraints.txt') - constraints_certbot = subprocess.check_output( - [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_certbot], - universal_newlines=True) - constraints_pipstrap = subprocess.check_output( - [sys.executable, os.path.join(repo_path, 'tools', 'strip_hashes.py'), reqs_pipstrap], - universal_newlines=True) - workdir = tempfile.mkdtemp() - try: - constraints_file_path = os.path.join(workdir, 'constraints.txt') - with open(constraints_file_path, 'a') as file_h: - file_h.write(constraints_pipstrap) - file_h.write(constraints_certbot) - file_h.write('pywin32=={0}'.format(PYWIN32_VERSION)) - yield constraints_file_path - finally: - shutil.rmtree(workdir) - - def _copy_assets(build_path, repo_path): print('Copy assets') if os.path.exists(build_path):