-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Build manylinux wheels with Zuul (#5386)
This adds the Zuul playbooks and role to build manylinux wheels for aarch64 and x86_64 (while aarch64 is the primary goal; it's good for the overall code to keep it flexible). It first builds an sdist from the checkout and then builds the wheels in the appropriate containers. Note this adds the jobs in the gate pipeline, which currently responds to Pull Requests, and the release pipeline, which responds to pushes to refs/tags/.* (see [1]). Note for results of jobs run against tags you will need to find the job directly from https://zuul.opendev.org/t/pyca/builds because there is nowhere to report the results as such (it could be configured to send an email). The wheels are published to the wheelhouse/ directory in the Zuul logs, which is also listed as an artifact on the build results page. [1] https://review.opendev.org/748323
- Loading branch information
Showing
7 changed files
with
245 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,13 @@ | ||
- project: | ||
check: | ||
jobs: | ||
- pyca-cryptography-build-wheel-arm64 | ||
- pyca-cryptography-build-wheel-x86_64 | ||
- pyca-cryptography-ubuntu-focal-py38-arm64 | ||
- pyca-cryptography-ubuntu-bionic-py36-arm64 | ||
- pyca-cryptography-centos-8-py36-arm64 | ||
- pyca-cryptography-centos-8-py27-arm64 | ||
release: | ||
jobs: | ||
- pyca-cryptography-build-wheel-arm64 | ||
- pyca-cryptography-build-wheel-x86_64 |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
- hosts: all | ||
tasks: | ||
|
||
- name: Build wheel | ||
include_role: | ||
name: build-wheel-manylinux |
1 change: 1 addition & 0 deletions
1
.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/README.rst
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
Build manylinux wheels for cryptography |
51 changes: 51 additions & 0 deletions
51
.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/files/build-wheels.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
#!/bin/bash -ex | ||
|
||
# Compile wheels | ||
cd /io | ||
|
||
mkdir -p wheelhouse.final | ||
|
||
for P in ${PYTHONS}; do | ||
|
||
PYBIN=/opt/python/${P}/bin | ||
|
||
"${PYBIN}"/python -m virtualenv .venv | ||
|
||
.venv/bin/pip install cffi six ipaddress "enum34; python_version < '3'" | ||
|
||
REGEX="cp3([0-9])*" | ||
if [[ "${PYBIN}" =~ $REGEX ]]; then | ||
PY_LIMITED_API="--py-limited-api=cp3${BASH_REMATCH[1]}" | ||
fi | ||
|
||
LDFLAGS="-L/opt/pyca/cryptography/openssl/lib" \ | ||
CFLAGS="-I/opt/pyca/cryptography/openssl/include -Wl,--exclude-libs,ALL" \ | ||
.venv/bin/python setup.py bdist_wheel $PY_LIMITED_API | ||
|
||
auditwheel repair --plat ${PLAT} -w wheelhouse/ dist/cryptography*.whl | ||
|
||
# Sanity checks | ||
# NOTE(ianw) : no execstack on aarch64, comes from | ||
# prelink, which was never supported. CentOS 8 does | ||
# have it separate, skip for now. | ||
if [[ "${PLAT}" != "manylinux2014_aarch64" ]]; then | ||
for f in wheelhouse/*.whl; do | ||
unzip $f -d execstack.check | ||
|
||
results=$(execstack execstack.check/cryptography/hazmat/bindings/*.so) | ||
count=$(echo "$results" | grep -c '^X' || true) | ||
if [ "$count" -ne 0 ]; then | ||
exit 1 | ||
fi | ||
rm -rf execstack.check | ||
done | ||
fi | ||
|
||
.venv/bin/pip install cryptography --no-index -f wheelhouse/ | ||
.venv/bin/python -c "from cryptography.hazmat.backends.openssl.backend import backend;print('Loaded: ' + backend.openssl_version_text());print('Linked Against: ' + backend._ffi.string(backend._lib.OPENSSL_VERSION_TEXT).decode('ascii'))" | ||
|
||
# Cleanup | ||
mv wheelhouse/* wheelhouse.final | ||
rm -rf .venv dist wheelhouse | ||
|
||
done |
145 changes: 145 additions & 0 deletions
145
.zuul.playbooks/playbooks/wheel/roles/build-wheel-manylinux/tasks/main.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,145 @@ | ||
# Wheel builds is a list of dicts, with keys | ||
# | ||
# platform: the manylinux platform name | ||
# image: the docker image to build in | ||
# pythons: list of pythons in the image to build wheels for | ||
- name: Sanity check build list | ||
assert: | ||
that: wheel_builds is defined | ||
|
||
- name: Ensure pip installed | ||
include_role: | ||
name: ensure-pip | ||
|
||
- name: Run ensure-docker | ||
include_role: | ||
name: ensure-docker | ||
|
||
- name: Workaround Linaro aarch64 cloud MTU issues | ||
# NOTE(ianw) : Docker default networking, the Linaro NAT setup and | ||
# *insert random things here* cause PMTU issues, resulting in hung | ||
# connections, particularly to fastly CDN (particularly annoying | ||
# because pypi and pythonhosted live behind that). Can remove after | ||
# upstream changes merge, or we otherwise find a solution in the | ||
# upstream cloud. | ||
# https://review.opendev.org/747062 | ||
# https://review.opendev.org/746833 | ||
# https://review.opendev.org/747064 | ||
when: ansible_architecture == 'aarch64' | ||
block: | ||
- name: Install jq | ||
package: | ||
name: jq | ||
state: present | ||
become: yes | ||
|
||
- name: Reset docker MTU | ||
shell: | | ||
jq --arg mtu 1400 '. + {mtu: $mtu|tonumber}' /etc/docker/daemon.json > /etc/docker/daemon.json.new | ||
cat /etc/docker/daemon.json.new | ||
mv /etc/docker/daemon.json.new /etc/docker/daemon.json | ||
service docker restart | ||
become: yes | ||
|
||
# We build an sdist of the checkout, and then build wheels from the | ||
# sdist. This ensures that nothing is left out of the sdist. | ||
- name: Install sdist required packages | ||
package: | ||
name: | ||
- build-essential | ||
- libssl-dev | ||
- libffi-dev | ||
- python3-dev | ||
become: yes | ||
when: ansible_distribution in ['Debian', 'Ubuntu'] | ||
|
||
- name: Create sdist | ||
command: | | ||
python3 setup.py sdist | ||
args: | ||
chdir: '{{ ansible_user_dir }}/{{ zuul.project.src_dir }}' | ||
|
||
- name: Find output file | ||
find: | ||
paths: '{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/dist' | ||
file_type: file | ||
patterns: "*.tar.gz" | ||
register: _sdist | ||
|
||
- assert: | ||
that: | ||
- _sdist.matched == 1 | ||
|
||
- name: Create a build area | ||
file: | ||
path: '{{ ansible_user_dir }}/build' | ||
state: directory | ||
|
||
- name: Create build area from sdist | ||
unarchive: | ||
src: '{{ _sdist.files[0].path }}' | ||
dest: '{{ ansible_user_dir }}/build' | ||
remote_src: yes | ||
|
||
- name: Find cryptography subdir from sdist build dir | ||
set_fact: | ||
_build_dir: "{{ ansible_user_dir }}/build/{{ _sdist.files[0].path | basename | replace('.tar.gz', '') }}" | ||
|
||
- name: Show _build_dir | ||
debug: | ||
var: _build_dir | ||
|
||
- name: Install build script | ||
copy: | ||
src: build-wheels.sh | ||
dest: '{{ _build_dir }}' | ||
mode: 0755 | ||
|
||
- name: Pre-pull containers | ||
command: >- | ||
docker pull {{ item.image }} | ||
become: yes | ||
loop: '{{ wheel_builds }}' | ||
|
||
- name: Run builds | ||
command: | | ||
docker run --rm \ | ||
-e PLAT={{ item.platform }} \ | ||
-e PYTHONS="{{ item.pythons | join(' ') }}" \ | ||
-v {{ _build_dir }}:/io \ | ||
{{ item.image }} \ | ||
/io/build-wheels.sh | ||
become: yes | ||
loop: '{{ wheel_builds }}' | ||
|
||
- name: Copy sdist to output | ||
synchronize: | ||
src: '{{ _sdist.files[0].path }}' | ||
dest: '{{ zuul.executor.log_root }}' | ||
mode: pull | ||
|
||
- name: Return sdist artifact | ||
zuul_return: | ||
data: | ||
zuul: | ||
artifacts: | ||
- name: '{{ _sdist.files[0].path | basename }}' | ||
url: 'sdist/{{ _sdist.files[0].path }}' | ||
metadata: | ||
type: sdist | ||
|
||
- name: Copy wheels to output | ||
synchronize: | ||
src: '{{ _build_dir }}/wheelhouse.final/' | ||
dest: '{{ zuul.executor.log_root }}/wheelhouse' | ||
mode: pull | ||
|
||
- name: Return wheelhouse artifact | ||
zuul_return: | ||
data: | ||
zuul: | ||
artifacts: | ||
- name: "Wheelhouse" | ||
url: "wheelhouse" | ||
metadata: | ||
type: wheelhouse |