Merge pull request #209 from Ousret/3.0

Version 3.0.0rc1
jawah · Oct 18, 2022 · 544595d · 544595d
2 parents 6602dae + 6367d53
commit 544595d
Show file tree

Hide file tree

Showing 38 changed files with 933 additions and 509 deletions.
diff --git a/.github/workflows/chardet-bc.yml b/.github/workflows/chardet-bc.yml
@@ -25,7 +25,8 @@ jobs:
         pip uninstall -y charset-normalizer
     - name: Install the package
       run: |
-        python setup.py install
+        python -m build
+        pip install ./dist/*.whl
     - name: Clone the complete dataset
       run: |
         git clone https://github.com/Ousret/char-dataset.git

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
diff --git a/.github/workflows/detector-coverage.yml b/.github/workflows/detector-coverage.yml
@@ -25,7 +25,8 @@ jobs:
         pip uninstall -y charset-normalizer
     - name: Install the package
       run: |
-        python setup.py install
+        python -m build
+        pip install ./dist/*.whl
     - name: Clone the complete dataset
       run: |
         git clone https://github.com/Ousret/char-dataset.git

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -28,7 +28,8 @@ jobs:
         pip uninstall -y charset-normalizer
     - name: Install the package
       run: |
-        python setup.py install
+        python -m build
+        pip install ./dist/*.whl
     - name: Clone the complete dataset
       run: |
         git clone https://github.com/Ousret/char-dataset.git

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -25,7 +25,8 @@ jobs:
         pip uninstall -y charset-normalizer
     - name: Install the package
       run: |
-        python setup.py install
+        python -m build
+        pip install ./dist/*.whl
     - name: Type checking (Mypy)
       run: |
         mypy --strict charset_normalizer

diff --git a/.github/workflows/mypyc-verify.yml b/.github/workflows/mypyc-verify.yml
@@ -0,0 +1,40 @@
+name: MYPYC Run
+
+on: [push, pull_request]
+
+jobs:
+  detection_coverage:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.6, 3.7, 3.8, 3.9, "3.10"]
+        os: [ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        pip install -U pip setuptools
+        pip install -r dev-requirements.txt
+        pip uninstall -y charset-normalizer
+    - name: Install the package
+      env:
+        CHARSET_NORMALIZER_USE_MYPYC: '1'
+      run: |
+        python -m build --no-isolation
+        pip install ./dist/*.whl
+    - name: Clone the complete dataset
+      run: |
+        git clone https://github.com/Ousret/char-dataset.git
+    - name: Coverage WITH preemptive
+      run: |
+        python ./bin/coverage.py --coverage 97 --with-preemptive
+    - name: Coverage WITHOUT preemptive
+      run: |
+        python ./bin/coverage.py --coverage 95
diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
@@ -25,7 +25,8 @@ jobs:
         pip uninstall -y charset-normalizer
     - name: Install the package
       run: |
-        python setup.py install
+        python -m build
+        pip install ./dist/*.whl
     - name: Clone the complete dataset
       run: |
         git clone https://github.com/Ousret/char-dataset.git

diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
@@ -29,7 +29,8 @@ jobs:
           pip uninstall -y charset-normalizer
       - name: Install the package
         run: |
-          python setup.py install
+          python -m build
+          pip install ./dist/*.whl
       - name: Type checking (Mypy)
         run: |
           mypy charset_normalizer
@@ -51,7 +52,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10" ]
+        python-version: [ 3.6, 3.7, 3.8, 3.9, "3.10", "3.11-dev" ]
         os: [ ubuntu-latest ]
 
     steps:
@@ -67,7 +68,8 @@ jobs:
           pip uninstall -y charset-normalizer
       - name: Install the package
         run: |
-          python setup.py install
+          python -m build
+          pip install ./dist/*.whl
       - name: Run tests
         run: |
           pytest
@@ -96,7 +98,8 @@ jobs:
           pip uninstall -y charset-normalizer
       - name: Install the package
         run: |
-          python setup.py install
+          python -m build
+          pip install ./dist/*.whl
       - name: Clone the complete dataset
         run: |
           git clone https://github.com/Ousret/char-dataset.git
@@ -136,7 +139,8 @@ jobs:
           pip uninstall -y charset-normalizer
       - name: Install the package
         run: |
-          python setup.py install
+          python -m build
+          pip install ./dist/*.whl
       - name: Clone the complete dataset
         run: |
           git clone https://github.com/Ousret/char-dataset.git
@@ -146,11 +150,92 @@ jobs:
       - name: Integration Tests with Requests
         run: |
           python ./bin/integration.py
+  universal-wheel:
+    runs-on: ubuntu-latest
+    needs:
+      - integration
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+      - name: Update pip, setuptools, wheel and twine
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+      - name: Build Wheel
+        env:
+          CHARSET_NORMALIZER_USE_MYPYC: '0'
+        run: python -m build
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist
+
+  build-wheels:
+    name: Build wheels on ${{ matrix.os }} ${{ matrix.qemu }}
+    runs-on: ${{ matrix.os }}-latest
+    needs: universal-wheel
+    strategy:
+      matrix:
+        os: [ ubuntu, windows, macos ]
+        qemu: [ '' ]
+        include:
+          # Split ubuntu job for the sake of speed-up
+          - os: ubuntu
+            qemu: aarch64
+          - os: ubuntu
+            qemu: ppc64le
+          - os: ubuntu
+            qemu: s390x
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Set up QEMU
+        if: ${{ matrix.qemu }}
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: all
+        id: qemu
+      - name: Prepare emulation
+        run: |
+          if [[ -n "${{ matrix.qemu }}" ]]; then
+            # Build emulated architectures only if QEMU is set,
+            # use default "auto" otherwise
+            echo "CIBW_ARCHS_LINUX=${{ matrix.qemu }}" >> $GITHUB_ENV
+          fi
+        shell: bash
+      - name: Setup Python
+        uses: actions/setup-python@v4
+      - name: Update pip, wheel, setuptools, build, twine
+        run: |
+          python -m pip install -U pip wheel setuptools build twine
+      - name: Build wheels
+        uses: pypa/cibuildwheel@2.10.2
+        env:
+          CIBW_BUILD_FRONTEND: "build"
+          CIBW_ARCHS_MACOS: x86_64 arm64 universal2
+          CIBW_ENVIRONMENT: CHARSET_NORMALIZER_USE_MYPYC='1'
+          CIBW_CONFIG_SETTINGS: "--no-isolation"
+          CIBW_BEFORE_BUILD: pip install -r dev-requirements.txt
+          CIBW_TEST_REQUIRES: pytest codecov pytest-cov
+          CIBW_TEST_COMMAND: pytest {package}/tests
+          CIBW_SKIP: pp*
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: ./wheelhouse/*.whl
+
   deploy:
 
     runs-on: ubuntu-latest
     needs:
-      - integration
+      - build-wheels
 
     steps:
     - uses: actions/checkout@v2
@@ -162,10 +247,17 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install setuptools wheel twine
-    - name: Build and publish
+    - name: Download disctributions
+      uses: actions/download-artifact@v3
+      with:
+        name: dist
+        path: dist
+    - name: Collected dists
+      run: |
+        tree dist
+    - name: Publish
       env:
         TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
         TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
       run: |
-        python setup.py sdist bdist_wheel
         twine upload dist/*
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -25,7 +25,8 @@ jobs:
         pip uninstall -y charset-normalizer
     - name: Install the package
       run: |
-        python setup.py install
+        python -m build --no-isolation
+        pip install ./dist/*.whl
     - name: Run tests
       run: |
         pytest

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,48 @@
 All notable changes to charset-normalizer will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [3.0.0rc1](https://github.com/Ousret/charset_normalizer/compare/3.0.0b2...3.0.0rc1) (2022-10-18)
+
+### Added
+- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results
+- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES
+- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio
+
+### Changed
+- Build with static metadata using 'build' frontend
+- Make the language detection stricter
+
+### Fixed
+- CLI with opt --normalize fail when using full path for files
+- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it
+
+### Removed
+- Coherence detector no longer return 'Simple English' instead return 'English'
+- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese'
+
+## [3.0.0b2](https://github.com/Ousret/charset_normalizer/compare/3.0.0b1...3.0.0b2) (2022-08-21)
+
+### Added
+- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl)
+
+### Removed
+- Breaking: Method `first()` and `best()` from CharsetMatch
+- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII)
+
+### Fixed
+- Sphinx warnings when generating the documentation
+
+## [3.0.0b1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...3.0.0b1) (2022-08-15)
+
+### Changed
+- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1
+
+### Removed
+- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches
+- Breaking: Top-level function `normalize`
+- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch
+- Support for the backport `unicodedata2`
+
 ## [2.1.1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...2.1.1) (2022-08-19)
 
 ### Deprecated