Move tests to pytest

* custom pytest collector and item to surface pyyaml data-driven tests as individual pytest unit tests * moved some true unit tests to pytest-native tests * deprecated `setup.py test` custom command * updated Makefile to use pytest * align test matrix with planned 7.x Python support
yaml · Nov 10, 2023 · a98fd60 · a98fd60
1 parent 155ec46
commit a98fd60
Show file tree

Hide file tree

Showing 606 changed files with 184 additions and 117 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -120,22 +120,16 @@ jobs:
     strategy:
       matrix:
         include:
-        - { platform: manylinux1, arch: x86_64, spec: cp36 }
-#        - { platform: manylinux1, arch: x86_64, spec: cp37 }
-#        - { platform: manylinux1, arch: x86_64, spec: cp38 }
+        - { platform: manylinux1, arch: x86_64, spec: cp38 }
 #        - { platform: manylinux1, arch: x86_64, spec: cp39 }
 #        - { platform: manylinux2014, arch: x86_64, spec: cp310 }
 #        - { platform: manylinux2014, arch: x86_64, spec: cp311 }
         - { platform: manylinux2014, arch: x86_64, spec: cp312 }
-#        - { platform: manylinux2014, arch: aarch64, spec: cp36 }
-#        - { platform: manylinux2014, arch: aarch64, spec: cp37 }
 #        - { platform: manylinux2014, arch: aarch64, spec: cp38 }
 #        - { platform: manylinux2014, arch: aarch64, spec: cp39 }
 #        - { platform: manylinux2014, arch: aarch64, spec: cp310 }
 #        - { platform: manylinux2014, arch: aarch64, spec: cp311 }
 #        - { platform: manylinux2014, arch: aarch64, spec: cp312 }
-#        - { platform: manylinux2014, arch: s390x, spec: cp36 }
-#        - { platform: manylinux2014, arch: s390x, spec: cp37 }
 #        - { platform: manylinux2014, arch: s390x, spec: cp38 }
 #        - { platform: manylinux2014, arch: s390x, spec: cp39 }
 #        - { platform: manylinux2014, arch: s390x, spec: cp310 }
@@ -176,7 +170,8 @@ jobs:
           LD_LIBRARY_PATH=libyaml/src/.libs
           PYYAML_FORCE_CYTHON=1
           PYYAML_FORCE_LIBYAML=1
-        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
+        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_TEST_REQUIRES: pytest
       run: |
         set -eux
         python3 -V
@@ -239,10 +234,6 @@ jobs:
     strategy:
       matrix:
         include:
-#        - spec: cp36-macosx_x86_64
-#          cibw_version: cibuildwheel==2.11.1
-#        - spec: cp37-macosx_x86_64
-#          cibw_version: cibuildwheel==2.11.1
 #        - spec: cp38-macosx_x86_64
 #          cibw_version: cibuildwheel==2.11.1
 #        - spec: cp39-macosx_x86_64
@@ -296,9 +287,12 @@ jobs:
         C_INCLUDE_PATH: libyaml/include
         CIBW_BUILD: ${{matrix.spec}}
         CIBW_BUILD_VERBOSITY: 1
-        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
+        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_TEST_REQUIRES: pytest
         LIBRARY_PATH: libyaml/src/.libs
         MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
+        PYYAML_FORCE_CYTHON: 1
+        PYYAML_FORCE_LIBYAML: 1
         SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
       run: |
         python3 -V
@@ -360,17 +354,6 @@ jobs:
     strategy:
       matrix:
         include:
-#        - platform: windows-2019
-#          build_arch: x64
-#          python_arch: x64
-#          spec: 3.6
-#        - platform: windows-2019
-#          build_arch: x64
-#          python_arch: x64
-#          spec: 3.7
-#        - platform: windows-2019
-#          build_arch: x64
-#          python_arch: x64
 #          spec: 3.8
 #        - platform: windows-2019
 #          build_arch: x64
@@ -387,15 +370,7 @@ jobs:
         - platform: windows-2019
           build_arch: x64
           python_arch: x64
-          spec: '3.12.0-rc.1'
-#        - platform: windows-2019
-#          build_arch: win32
-#          python_arch: x86
-#          spec: 3.6
-#        - platform: windows-2019
-#          build_arch: win32
-#          python_arch: x86
-#          spec: 3.7
+          spec: '3.12'
 #        - platform: windows-2019
 #          build_arch: win32
 #          python_arch: x86
@@ -415,7 +390,7 @@ jobs:
         - platform: windows-2019
           build_arch: win32
           python_arch: x86
-          spec: '3.12.0-rc.1'
+          spec: '3.12'
     steps:
     # autocrlf screws up tests under Windows
     - name: Set git to use LF
@@ -458,8 +433,8 @@ jobs:
         build bdist_wheel
 
         # run tests on built wheel
-        python -m pip install dist/*.whl
-        python tests/lib/test_all.py
+        python -m pip install dist/*.whl pytest
+        python -I -m pytest
 
     - name: Upload artifacts
       uses: actions/upload-artifact@v3

diff --git a/.github/workflows/manual_artifact_build.yaml b/.github/workflows/manual_artifact_build.yaml
@@ -1,7 +1,7 @@
 ---
 # This is the actual artifact build/release workflow. This workflow exists temporarily
 # because GHA doesn't support a dynamic/conditional matrix. Ensure changes are synced with ci.yaml.
-name: Manual Artifact Build
+name: PyYAML CI
 
 on:
 #  push:
@@ -76,7 +76,6 @@ jobs:
         - { platform: manylinux2014, arch: aarch64 }
         - { platform: manylinux2014, arch: s390x }
         - { platform: musllinux_1_1, arch: x86_64 }
-
     env:
       DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}}
     steps:
@@ -119,22 +118,16 @@ jobs:
     strategy:
       matrix:
         include:
-        - { platform: manylinux1, arch: x86_64, spec: cp36 }
-        - { platform: manylinux1, arch: x86_64, spec: cp37 }
         - { platform: manylinux1, arch: x86_64, spec: cp38 }
         - { platform: manylinux1, arch: x86_64, spec: cp39 }
         - { platform: manylinux2014, arch: x86_64, spec: cp310 }
         - { platform: manylinux2014, arch: x86_64, spec: cp311 }
         - { platform: manylinux2014, arch: x86_64, spec: cp312 }
-        - { platform: manylinux2014, arch: aarch64, spec: cp36 }
-        - { platform: manylinux2014, arch: aarch64, spec: cp37 }
         - { platform: manylinux2014, arch: aarch64, spec: cp38 }
         - { platform: manylinux2014, arch: aarch64, spec: cp39 }
         - { platform: manylinux2014, arch: aarch64, spec: cp310 }
         - { platform: manylinux2014, arch: aarch64, spec: cp311 }
         - { platform: manylinux2014, arch: aarch64, spec: cp312 }
-        - { platform: manylinux2014, arch: s390x, spec: cp36 }
-        - { platform: manylinux2014, arch: s390x, spec: cp37 }
         - { platform: manylinux2014, arch: s390x, spec: cp38 }
         - { platform: manylinux2014, arch: s390x, spec: cp39 }
         - { platform: manylinux2014, arch: s390x, spec: cp310 }
@@ -175,7 +168,8 @@ jobs:
           LD_LIBRARY_PATH=libyaml/src/.libs
           PYYAML_FORCE_CYTHON=1
           PYYAML_FORCE_LIBYAML=1
-        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
+        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_TEST_REQUIRES: pytest
       run: |
         set -eux
         python3 -V
@@ -238,10 +232,6 @@ jobs:
     strategy:
       matrix:
         include:
-        - spec: cp36-macosx_x86_64
-          cibw_version: cibuildwheel==2.11.1
-        - spec: cp37-macosx_x86_64
-          cibw_version: cibuildwheel==2.11.1
         - spec: cp38-macosx_x86_64
           cibw_version: cibuildwheel==2.11.1
         - spec: cp39-macosx_x86_64
@@ -295,9 +285,12 @@ jobs:
         C_INCLUDE_PATH: libyaml/include
         CIBW_BUILD: ${{matrix.spec}}
         CIBW_BUILD_VERBOSITY: 1
-        CIBW_TEST_COMMAND: cd {project}; python tests/lib/test_all.py
+        CIBW_TEST_COMMAND: cd {project}; pytest
+        CIBW_TEST_REQUIRES: pytest
         LIBRARY_PATH: libyaml/src/.libs
         MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target || '10.9' }}
+        PYYAML_FORCE_CYTHON: 1
+        PYYAML_FORCE_LIBYAML: 1
         SDKROOT: ${{ matrix.sdkroot || 'macosx' }}
       run: |
         python3 -V
@@ -359,17 +352,6 @@ jobs:
     strategy:
       matrix:
         include:
-        - platform: windows-2019
-          build_arch: x64
-          python_arch: x64
-          spec: 3.6
-        - platform: windows-2019
-          build_arch: x64
-          python_arch: x64
-          spec: 3.7
-        - platform: windows-2019
-          build_arch: x64
-          python_arch: x64
           spec: 3.8
         - platform: windows-2019
           build_arch: x64
@@ -386,15 +368,7 @@ jobs:
         - platform: windows-2019
           build_arch: x64
           python_arch: x64
-          spec: '3.12.0-rc.1'
-        - platform: windows-2019
-          build_arch: win32
-          python_arch: x86
-          spec: 3.6
-        - platform: windows-2019
-          build_arch: win32
-          python_arch: x86
-          spec: 3.7
+          spec: '3.12'
         - platform: windows-2019
           build_arch: win32
           python_arch: x86
@@ -414,7 +388,7 @@ jobs:
         - platform: windows-2019
           build_arch: win32
           python_arch: x86
-          spec: '3.12.0-rc.1'
+          spec: '3.12'
     steps:
     # autocrlf screws up tests under Windows
     - name: Set git to use LF
@@ -457,8 +431,8 @@ jobs:
         build bdist_wheel
 
         # run tests on built wheel
-        python -m pip install dist/*.whl
-        python tests/lib/test_all.py
+        python -m pip install dist/*.whl pytest
+        python -I -m pytest
 
     - name: Upload artifacts
       uses: actions/upload-artifact@v3

diff --git a/.gitignore b/.gitignore
@@ -16,4 +16,4 @@ _pycache_/*
 
 # local IDE state
 /.idea/*
-
+/.tox/*
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -2,6 +2,6 @@ include CHANGES README LICENSE Makefile pyproject.toml setup.py
 recursive-include lib/yaml *.py
 recursive-include lib/_yaml *.py
 recursive-include examples *.py *.cfg *.yaml
-recursive-include tests/data *
-recursive-include tests/lib *.py
+recursive-include tests/legacy_tests/ *.py
+recursive-include tests/legacy_tests/data *
 recursive-include yaml *
diff --git a/Makefile b/Makefile
@@ -24,28 +24,25 @@ installext:
 	${PYTHON} setup.py --with-libyaml install ${PARAMETERS}
 
 test: build
-	${PYTHON} tests/lib/test_build.py ${TEST}
+	PYYAML_FORCE_LIBYAML=0 ${PYTHON} -I -m pytest
 
 testext: buildext
-	${PYTHON} tests/lib/test_build_ext.py ${TEST}
+	PYYAML_FORCE_LIBYAML=1 ${PYTHON} -I -m pytest
 
 testall:
-	${PYTHON} setup.py test
+	${PYTHON} -m pytest
 
 dist:
 	@# No longer uploading a zip file to pypi
 	@# ${PYTHON} setup.py --with-libyaml sdist --formats=zip,gztar
 	${PYTHON} setup.py --with-libyaml sdist --formats=gztar
 
-windist:
-	${PYTHON} setup.py --with-libyaml bdist_wininst
-
 clean:
 	${PYTHON} setup.py --with-libyaml clean -a
 	rm -fr \
 	    dist/ \
 	    lib/PyYAML.egg-info/ \
 	    lib/yaml/__pycache__/ \
-	    tests/lib/__pycache__/ \
-	    yaml/_yaml.c \
-
+	    tests/__pycache__/ \
+	    tests/legacy_tests/__pycache__/ \
+	    yaml/_yaml.c
diff --git a/setup.py b/setup.py
@@ -250,6 +250,7 @@ def finalize_options(self):
         pass
 
     def run(self):
+        warnings.warn('Running tests via `setup.py test` is deprecated and will be removed in a future release. Use `pytest` instead to ensure that the complete test suite is run.', DeprecationWarning)
         build_cmd = self.get_finalized_command('build')
         build_cmd.run()
 
@@ -263,7 +264,7 @@ def run(self):
             temp_test_path = pathlib.Path(tempdir.name) / 'pyyaml'
             shutil.copytree(build_cmd.build_lib, temp_test_path)
             sys.path.insert(0, str(temp_test_path))
-            sys.path.insert(0, 'tests/lib')
+            sys.path.insert(0, 'tests/legacy_tests')
 
             import test_all
             if not test_all.main([]):

diff --git a/tests/lib/canonical.py → tests/legacy_tests/canonical.py b/tests/lib/canonical.py → tests/legacy_tests/canonical.py