Merge pull request #1051 from altendky/9217-altendky-cibuildwheel

Author: altendky

Reviewer: glyph

Fixes: ticket:9217

Build wheels in CI environments so that process is reproducible.

.circleci/config.yml

@@ -15,35 +15,86 @@
 #
 version: 2
 
+aliases:
+  - &preparation_run_group
+    run:
+      name: Prepare the virtualenv.
+      command: |
+        "$PYTHON" --version
+        "$PYTHON" -m pip --version
+        "$PYTHON" -m pip install -q --user --ignore-installed --upgrade virtualenv
+        "$PYTHON" -m virtualenv -p "$PYTHON" venv
+        venv/bin/pip install -U pip
+        venv/bin/python -m pip install tox
+        venv/bin/python -m pip freeze
+        venv/bin/python --version
+
 jobs:
+  osx-wheels:
+    working_directory: ~/osx-wheels
+    macos:
+      xcode: "10.0.0"
+    environment:
+      PYTHON: python3
+    steps:
+      # Get the source.
+      - checkout
+
+      - <<: *preparation_run_group
+
+      - run:
+          name: Build the OSX wheels.
+          command: |
+            venv/bin/tox -r -e wheels
+
+      - store_artifacts:
+          path: wheelhouse/
+
+  linux-wheels:
+    working_directory: ~/linux-wheels
+    docker:
+      - image: circleci/python:3.6
+    environment:
+      PYTHON: python3
+    steps:
+      # Get the source.
+      - checkout
+      - setup_remote_docker
+
+      - <<: *preparation_run_group
+
+      - run:
+          name: Build the Linux wheels.
+          command: |
+            venv/bin/tox -r -e wheels
+
+      - store_artifacts:
+          path: wheelhouse/
+
   static_checkers:
     working_directory: ~/static-checkers
     docker:
       - image: circleci/python:2.7
+    environment:
+      PYTHON: python2
     steps:
       # Get the source.
       - checkout
 
-      - run:
-          name: Prepare the environment.
-          command: |
-            python --version
-            pip install -q --user --ignore-installed --upgrade virtualenv
-            pip install -q tox --user
-            echo 'export PATH=~/.local/bin:$PATH' >> $BASH_ENV
+      - <<: *preparation_run_group
 
       # Run each checker in a separate step, but always run all steps to get
       # a the results from each step.
       - run:
           name: Check newsfragment.
           when: always
           command: |
-            tox -r -e newsfragment
+            venv/bin/tox -r -e newsfragment
       - run:
           name: Check manifest-checker.
           when: always
           command: |
-            tox -r -e manifest-checker
+            venv/bin/tox -r -e manifest-checker
 
   #
   # Documentation tests are slow so we execute them in a separate job.
@@ -52,27 +103,23 @@ jobs:
     working_directory: ~/documentation
     docker:
       - image: circleci/python:2.7
+    environment:
+      PYTHON: python2
     steps:
       # Get the source.
       - checkout
 
-      - run:
-          name: Prepare the environment.
-          command: |
-            python --version
-            pip install -q --user --ignore-installed --upgrade virtualenv
-            pip install -q tox --user
-            echo 'export PATH=~/.local/bin:$PATH' >> $BASH_ENV
+      - <<: *preparation_run_group
 
       - run:
           name: Check the narrative documentation.
           command: |
-            tox -r -e narrativedocs
+            venv/bin/tox -r -e narrativedocs
 
       - run:
           name: Check the API documentation.
           command: |
-            tox -r -e apidocs
+            venv/bin/tox -r -e apidocs
 
   #
   # MacOS with Python2.7 and default reactor.
@@ -137,6 +184,11 @@ workflows:
     jobs:
       - static_checkers
       - documentation
+      - linux-wheels
+      - osx-wheels:
+          requires:
+            - static_checkers
+            - documentation
       - macos_py27_default_reactor:
           requires:
             - static_checkers

.travis.yml

@@ -5,7 +5,7 @@
 # way to share the same process between Travis-CI and Buildbot.
 #
 language: python
-sudo: false
+dist: xenial
 
 
 # Only run tests on push on a few branches.
@@ -23,6 +23,9 @@ env:
 
 matrix:
   include:
+    - services:
+        - docker
+      env: PIP=pip TOXENV=wheels
     - python: 2.7
       env: TOXENV=py27-alldeps-withcov-posix,codecov-publish
     - python: 2.7
@@ -35,16 +38,12 @@ matrix:
       env: TOXENV=py36-alldeps-withcov-posix,codecov-publish
     - python: 3.7
       env: TOXENV=py37-alldeps-withcov-posix,codecov-publish
-      dist: xenial
-      sudo: true
     - python: 3.5
       env: TOXENV=lint
     # We need a builder without IPv6. This is going to be slower than all the
     # others, but that's ok.
     - python: 3.6
       env: TOXENV=py36-alldeps-withcov-posix,codecov-publish DISABLE_IPV6=yes
-      sudo: true
-
 
 addons:
   apt:

admin/dump_all_version_info.py

@@ -0,0 +1,7 @@
+import sys
+
+print(sys.prefix)
+print(sys.exec_prefix)
+print(sys.executable)
+print(sys.version)
+print(sys.platform)

appveyor.yml

@@ -34,6 +34,11 @@ environment:
   # a later point release.
   # See: http://www.appveyor.com/docs/installed-software#python
   matrix:
+    - PYTHON_HOME: C:\\PYTHON36-x64
+      PYTHON_VERSION: "3.6"
+      PYTHON_ARCH: "64"
+      TOXENV: wheels
+
     - PYTHON_HOME: C:\\Python27-x64
       PYTHON_VERSION: "2.7"
       PYTHON_ARCH: "64"
@@ -103,7 +108,12 @@ install:
   - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
 
   # The build system itself is handled by tox.
+  - "python -m pip install -U pip"
   - "python -m pip install tox"
 
 test_script:
   - "%CMD_IN_ENV% python -m tox -e %TOXENV%"
+
+artifacts:
+  - path: "wheelhouse\\*.whl"
+    name: Wheels

tox.ini

@@ -64,29 +64,33 @@ deps =
     lint: diff-cover==0.9.12
     lint: pycodestyle
 
+    wheels: cibuildwheel==0.10.0
+
 ; All environment variables are passed.
 passenv = *
 
 setenv =
-   ; Enable sub-processes coverage reports and store coverage reports in a
-   ; known location.
-   COVERAGE_PROCESS_START = {toxinidir}/.coveragerc
-   COVERAGE_FILE = {toxinidir}/.coverage
-   # Help tests know where the base directory is.
-   TOX_INI_DIR = {toxinidir}
+    ; Enable sub-processes coverage reports and store coverage reports in a
+    ; known location.
+    COVERAGE_PROCESS_START = {toxinidir}/.coveragerc
+    COVERAGE_FILE = {toxinidir}/.coverage
+    # Help tests know where the base directory is.
+    TOX_INI_DIR = {toxinidir}
+
+    ; Skip twistedchecker warnings that are not yet required in the required run
+    lint: TWISTEDCHECKER_SKIP_WARNINGS = W9208,C0302,C0103,C9302
 
-   ; Skip twistedchecker warnings that are not yet required in the required run
-   lint: TWISTEDCHECKER_SKIP_WARNINGS = W9208,C0302,C0103,C9302
+    ; Configure cibuildwheel to build wheels for:
+    ;       CPython 2.7, 3.4, 3.5, 3.6, and 3.7.
+    ; This includes variants of bit depth, unicode width, etc.
+    wheels: CIBW_BUILD=cp27-* cp34-* cp35-* cp36-* cp37-*
 
 commands =
     ;
     ; Display information about Python interpreter
     ; which will be used in subsequent steps
     ;
-    python -c "import sys; print(sys.prefix)"
-    python -c "import sys; print(sys.exec_prefix)"
-    python -c "import sys; print(sys.executable)"
-    python --version
+    python {toxinidir}/admin/dump_all_version_info.py
 
     # We need to define nodeps *somewhere* so tox recognises it
     nodeps: python -c "print('No dependencies installed...')"
@@ -121,6 +125,10 @@ commands =
 
     manifest-checker: check-manifest --ignore "docs/_build*,docs/historic/*,admin*,bin/admin*,twisted/topfiles/*.Old"
 
+    wheels: cibuildwheel --output-dir {toxinidir}/wheelhouse {toxinidir}
+    wheels: python setup.py sdist --formats=gztar,zip --dist-dir={toxinidir}/wheelhouse
+    wheels: ls -l {toxinidir}/wheelhouse
+
 [testenv:apidocs]
 # Epydoc and pydoctor currently on work on Python 2
 basepython=python2.7