Merge branch 'main' into add-cygwin-to-ci

.ci/after_success.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # gather the coverage data
-pip3 install codecov
+python3 -m pip install codecov
 if [[ $MATRIX_DOCKER ]]; then
   coverage xml --ignore-errors
 else

.ci/install.sh

@@ -19,7 +19,7 @@ set -e
 
 sudo apt-get -qq install libfreetype6-dev liblcms2-dev python3-tk\
                          ghostscript libffi-dev libjpeg-turbo-progs libopenjp2-7-dev\
-                         cmake imagemagick libharfbuzz-dev libfribidi-dev
+                         cmake meson imagemagick libharfbuzz-dev libfribidi-dev
 
 python3 -m pip install --upgrade pip
 python3 -m pip install --upgrade wheel

.github/workflows/test-docker.yml

@@ -22,6 +22,7 @@ jobs:
           centos-8-amd64,
           centos-stream-8-amd64,
           debian-10-buster-x86,
+          debian-11-bullseye-x86,
           fedora-34-amd64,
           fedora-35-amd64,
           ubuntu-18.04-bionic-amd64,

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/psf/black
-    rev: 911470a610e47d9da5ea938b0887c3df62819b85  # frozen: 21.9b0
+    rev: f1d4e742c91dd5179d742b0db9293c4472b765f8  # frozen: 21.12b0
     hooks:
       - id: black
         args: ["--target-version", "py37"]
@@ -9,12 +9,12 @@ repos:
         types: []
 
   - repo: https://github.com/PyCQA/isort
-    rev: fd5ba70665a37ec301a1f714ed09336048b3be63  # frozen: 5.9.3
+    rev: c5e8fa75dda5f764d20f66a215d71c21cfa198e1  # frozen: 5.10.1
     hooks:
       - id: isort
 
   - repo: https://github.com/asottile/yesqa
-    rev: 644ede78511c02fc6f8e03e014cc1ddcfbf1e1f5  # frozen: v1.2.3
+    rev: 35cf7dc24fa922927caded7a21b2a8cb04bf8e10  # frozen: v1.3.0
     hooks:
       - id: yesqa
 
@@ -25,7 +25,7 @@ repos:
         exclude: (Makefile$|\.bat$|\.cmake$|\.eps$|\.fits$|\.opt$)
 
   - repo: https://github.com/PyCQA/flake8
-    rev: dcd740bc0ebaf2b3d43e59a0060d157c97de13f3  # frozen: 3.9.2
+    rev: cbeb4c9c4137cff1568659fcc48e8b85cddd0c8d  # frozen: 4.0.1
     hooks:
       - id: flake8
         additional_dependencies: [flake8-2020, flake8-implicit-str-concat]
@@ -37,7 +37,7 @@ repos:
       - id: rst-backticks
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: 38b88246ccc552bffaaf54259d064beeee434539  # frozen: v4.0.1
+    rev: 8fe62d14e0b4d7d845a7022c5c2c3ae41bdd3f26  # frozen: v4.1.0
     hooks:
       - id: check-merge-conflict
       - id: check-yaml

CHANGES.rst

@@ -2,9 +2,72 @@
 Changelog (Pillow)
 ==================
 
-9.0.0 (unreleased)
+9.1.0 (unreleased)
 ------------------
 
+- Raise an error when performing a negative crop #5972
+  [radarhere, hugovk]
+
+- Deprecated show_file "file" argument in favour of "path" #5959
+  [radarhere]
+
+- Fixed SPIDER images for use with Bio-formats library #5956
+  [radarhere]
+
+- Ensure duplicated file pointer is closed #5946
+  [radarhere]
+
+- Added specific error if ImagePath coordinate type is incorrect #5942
+  [radarhere]
+
+- Return an empty bytestring from tobytes() for an empty image #5938
+  [radarhere]
+
+- Remove readonly from Image.__eq__ #5930
+  [hugovk]
+
+9.0.0 (2022-01-02)
+------------------
+
+- Restrict builtins for ImageMath.eval(). CVE-2022-22817 #5923
+  [radarhere]
+
+- Ensure JpegImagePlugin stops at the end of a truncated file #5921
+  [radarhere]
+
+- Fixed ImagePath.Path array handling. CVE-2022-22815, CVE-2022-22816 #5920
+  [radarhere]
+
+- Remove consecutive duplicate tiles that only differ by their offset #5919
+  [radarhere]
+
+- Improved I;16 operations on big endian #5901
+  [radarhere]
+
+- Limit quantized palette to number of colors #5879
+  [radarhere]
+
+- Fixed palette index for zeroed color in FASTOCTREE quantize #5869
+  [radarhere]
+
+- When saving RGBA to GIF, make use of first transparent palette entry #5859
+  [radarhere]
+
+- Pass SAMPLEFORMAT to libtiff #5848
+  [radarhere]
+
+- Added rounding when converting P and PA #5824
+  [radarhere]
+
+- Improved putdata() documentation and data handling #5910
+  [radarhere]
+
+- Exclude carriage return in PDF regex to help prevent ReDoS #5912
+  [hugovk]
+
+- Fixed freeing pointer in ImageDraw.Outline.transform #5909
+  [radarhere]
+
 - Added ImageShow support for xdg-open #5897
   [m-shinder, radarhere]
 

LICENSE

@@ -5,7 +5,7 @@ The Python Imaging Library (PIL) is
 
 Pillow is the friendly PIL fork. It is
 
-    Copyright © 2010-2021 by Alex Clark and contributors
+    Copyright © 2010-2022 by Alex Clark and contributors
 
 Like PIL, Pillow is licensed under the open source HPND License:
 

Makefile

@@ -105,14 +105,14 @@ test:
 
 .PHONY: valgrind
 valgrind:
-	python3 -c "import pytest_valgrind" || pip3 install pytest-valgrind
+	python3 -c "import pytest_valgrind" || python3 -m pip install pytest-valgrind
 	PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp --leak-check=no \
             --log-file=/tmp/valgrind-output \
             python3 -m pytest --no-memcheck -vv --valgrind --valgrind-log=/tmp/valgrind-output
 
 .PHONY: readme
 readme:
-	python3 setup.py --long-description | markdown2 > .long-description.html && open .long-description.html
+	markdown2 README.md > .long-description.html && open .long-description.html
 
 
 .PHONY: lint

README.md

@@ -24,16 +24,19 @@ As of 2019, Pillow development is
     <tr>
         <th>tests</th>
         <td>
-            <a href="https://github.com/python-pillow/Pillow/actions?query=workflow%3ALint"><img
+            <a href="https://github.com/python-pillow/Pillow/actions/workflows/lint.yml"><img
                 alt="GitHub Actions build status (Lint)"
                 src="https://github.com/python-pillow/Pillow/workflows/Lint/badge.svg"></a>
-            <a href="https://github.com/python-pillow/Pillow/actions?query=workflow%3ATest"><img
+            <a href="https://github.com/python-pillow/Pillow/actions/workflows/test.yml"><img
                 alt="GitHub Actions build status (Test Linux and macOS)"
                 src="https://github.com/python-pillow/Pillow/workflows/Test/badge.svg"></a>
-            <a href="https://github.com/python-pillow/Pillow/actions?query=workflow%3A%22Test+Windows%22"><img
+            <a href="https://github.com/python-pillow/Pillow/actions/workflows/test-windows.yml"><img
                 alt="GitHub Actions build status (Test Windows)"
                 src="https://github.com/python-pillow/Pillow/workflows/Test%20Windows/badge.svg"></a>
-            <a href="https://github.com/python-pillow/Pillow/actions?query=workflow%3A%22Test+Docker%22"><img
+            <a href="https://github.com/python-pillow/Pillow/actions/workflows/test-mingw.yml"><img
+                alt="GitHub Actions build status (Test MinGW)"
+                src="https://github.com/python-pillow/Pillow/workflows/Test%20MinGW/badge.svg"></a>
+            <a href="https://github.com/python-pillow/Pillow/actions/workflows/test-docker.yml"><img
                 alt="GitHub Actions build status (Test Docker)"
                 src="https://github.com/python-pillow/Pillow/workflows/Test%20Docker/badge.svg"></a>
             <a href="https://ci.appveyor.com/project/python-pillow/Pillow"><img
@@ -42,10 +45,10 @@ As of 2019, Pillow development is
             <a href="https://github.com/python-pillow/pillow-wheels/actions"><img
                 alt="GitHub Actions wheels build status (Wheels)"
                 src="https://github.com/python-pillow/pillow-wheels/workflows/Wheels/badge.svg"></a>
-            <a href="https://travis-ci.com/github/python-pillow/pillow-wheels"><img
+            <a href="https://app.travis-ci.com/github/python-pillow/pillow-wheels"><img
                 alt="Travis CI wheels build status (aarch64)"
                 src="https://img.shields.io/travis/com/python-pillow/pillow-wheels/main.svg?label=aarch64%20wheels"></a>
-            <a href="https://codecov.io/gh/python-pillow/Pillow"><img
+            <a href="https://app.codecov.io/gh/python-pillow/Pillow"><img
                 alt="Code coverage"
                 src="https://codecov.io/gh/python-pillow/Pillow/branch/main/graph/badge.svg"></a>
             <a href="https://github.com/python-pillow/Pillow/actions/workflows/tidelift.yml"><img

Tests/helper.py

@@ -30,7 +30,6 @@ def upload(a, b):
             a.show()
             b.show()
 
-
 elif "GITHUB_ACTIONS" in os.environ:
     HAS_UPLOADER = True
 
@@ -44,7 +43,6 @@ def upload(a, b):
             b.save(os.path.join(tmpdir, "b.png"))
             return tmpdir
 
-
 else:
     try:
         import test_image_results

Tests/oss-fuzz/build.sh

@@ -22,7 +22,7 @@ for fuzzer in $(find $SRC -name 'fuzz_*.py'); do
   fuzzer_basename=$(basename -s .py $fuzzer)
   fuzzer_package=${fuzzer_basename}.pkg
   pyinstaller \
-      --add-binary /usr/local/lib/libjpeg.so.9:. \
+      --add-binary /usr/local/lib/libjpeg.so.62.3.0:. \
       --add-binary /usr/local/lib/libfreetype.so.6:. \
       --add-binary /usr/local/lib/liblcms2.so.2:. \
       --add-binary /usr/local/lib/libopenjp2.so.7:. \

Tests/test_decompression_bomb.py

@@ -86,21 +86,12 @@ def testEnlargeCrop(self):
             pytest.warns(Image.DecompressionBombWarning, src.crop, box)
 
     def test_crop_decompression_checks(self):
-
         im = Image.new("RGB", (100, 100))
 
-        good_values = ((-9999, -9999, -9990, -9990), (-999, -999, -990, -990))
-
-        warning_values = ((-160, -160, 99, 99), (160, 160, -99, -99))
-
-        error_values = ((-99909, -99990, 99999, 99999), (99909, 99990, -99999, -99999))
-
-        for value in good_values:
+        for value in ((-9999, -9999, -9990, -9990), (-999, -999, -990, -990)):
             assert im.crop(value).size == (9, 9)
 
-        for value in warning_values:
-            pytest.warns(Image.DecompressionBombWarning, im.crop, value)
+        pytest.warns(Image.DecompressionBombWarning, im.crop, (-160, -160, 99, 99))
 
-        for value in error_values:
-            with pytest.raises(Image.DecompressionBombError):
-                im.crop(value)
+        with pytest.raises(Image.DecompressionBombError):
+            im.crop((-99909, -99990, 99999, 99999))

Tests/test_file_bmp.py

@@ -123,3 +123,10 @@ def test_rgba_bitfields():
         im = Image.merge("RGB", (r, g, b))
 
     assert_image_equal_tofile(im, "Tests/images/bmp/q/rgb32bf-xbgr.bmp")
+
+
+def test_offset():
+    # This image has been hexedited
+    # to exclude the palette size from the pixel data offset
+    with Image.open("Tests/images/pal8_offset.bmp") as im:
+        assert_image_equal_tofile(im, "Tests/images/bmp/g/pal8.bmp")

Tests/test_file_gif.py

@@ -957,3 +957,13 @@ def test_missing_background():
     with Image.open("Tests/images/missing_background.gif") as im:
         im.seek(1)
         assert_image_equal_tofile(im, "Tests/images/missing_background_first_frame.png")
+
+
+def test_saving_rgba(tmp_path):
+    out = str(tmp_path / "temp.gif")
+    with Image.open("Tests/images/transparent.png") as im:
+        im.save(out)
+
+    with Image.open(out) as reloaded:
+        reloaded_rgba = reloaded.convert("RGBA")
+        assert reloaded_rgba.load()[0, 0][3] == 0

Tests/test_file_jpeg.py

@@ -870,6 +870,30 @@ def test_getxmp(self):
             with Image.open("Tests/images/hopper.jpg") as im:
                 assert im.getxmp() == {}
 
+    @pytest.mark.timeout(timeout=1)
+    def test_eof(self):
+        # Even though this decoder never says that it is finished
+        # the image should still end when there is no new data
+        class InfiniteMockPyDecoder(ImageFile.PyDecoder):
+            def decode(self, buffer):
+                return 0, 0
+
+        decoder = InfiniteMockPyDecoder(None)
+
+        def closure(mode, *args):
+            decoder.__init__(mode, *args)
+            return decoder
+
+        Image.register_decoder("INFINITE", closure)
+
+        with Image.open(TEST_FILE) as im:
+            im.tile = [
+                ("INFINITE", (0, 0, 128, 128), 0, ("RGB", 0, 1)),
+            ]
+            ImageFile.LOAD_TRUNCATED_IMAGES = True
+            im.load()
+            ImageFile.LOAD_TRUNCATED_IMAGES = False
+
 
 @pytest.mark.skipif(not is_win32(), reason="Windows only")
 @skip_unless_feature("jpg")

Tests/test_file_jpeg2k.py

@@ -10,7 +10,6 @@
     assert_image_equal,
     assert_image_similar,
     assert_image_similar_tofile,
-    is_big_endian,
     skip_unless_feature,
 )
 
@@ -234,13 +233,11 @@ def test_16bit_monochrome_has_correct_mode():
         assert jp2.mode == "I;16"
 
 
-@pytest.mark.xfail(is_big_endian(), reason="Fails on big-endian")
 def test_16bit_monochrome_jp2_like_tiff():
     with Image.open("Tests/images/16bit.cropped.tif") as tiff_16bit:
         assert_image_similar_tofile(tiff_16bit, "Tests/images/16bit.cropped.jp2", 1e-3)
 
 
-@pytest.mark.xfail(is_big_endian(), reason="Fails on big-endian")
 def test_16bit_monochrome_j2k_like_tiff():
     with Image.open("Tests/images/16bit.cropped.tif") as tiff_16bit:
         assert_image_similar_tofile(tiff_16bit, "Tests/images/16bit.cropped.j2k", 1e-3)

Tests/test_file_libtiff.py

@@ -9,7 +9,7 @@
 import pytest
 
 from PIL import Image, ImageFilter, TiffImagePlugin, TiffTags, features
-from PIL.TiffImagePlugin import STRIPOFFSETS, SUBIFD
+from PIL.TiffImagePlugin import SAMPLEFORMAT, STRIPOFFSETS, SUBIFD
 
 from .helper import (
     assert_image_equal,
@@ -825,6 +825,17 @@ def test_sampleformat(self):
 
             assert_image_equal_tofile(im, "Tests/images/copyleft.png", mode="RGB")
 
+    def test_sampleformat_write(self, tmp_path):
+        im = Image.new("F", (1, 1))
+        out = str(tmp_path / "temp.tif")
+        TiffImagePlugin.WRITE_LIBTIFF = True
+        im.save(out)
+        TiffImagePlugin.WRITE_LIBTIFF = False
+
+        with Image.open(out) as reloaded:
+            assert reloaded.mode == "F"
+            assert reloaded.getexif()[SAMPLEFORMAT] == 3
+
     def test_lzw(self):
         with Image.open("Tests/images/hopper_lzw.tif") as im:
             assert im.mode == "RGB"

Tests/test_file_pdf.py

@@ -313,8 +313,9 @@ def test_pdf_append_to_bytesio():
 
 
 @pytest.mark.timeout(1)
-def test_redos():
-    malicious = b" trailer<<>>" + b"\n" * 3456
+@pytest.mark.parametrize("newline", (b"\r", b"\n"))
+def test_redos(newline):
+    malicious = b" trailer<<>>" + newline * 3456
 
     # This particular exception isn't relevant here.
     # The important thing is it doesn't timeout, cause a ReDoS (CVE-2021-25292).

Tests/test_file_png.py

@@ -13,7 +13,6 @@
     assert_image_equal,
     assert_image_equal_tofile,
     hopper,
-    is_big_endian,
     is_win32,
     mark_if_feature_version,
     skip_unless_feature,
@@ -77,7 +76,6 @@ def get_chunks(self, filename):
                     png.crc(cid, s)
         return chunks
 
-    @pytest.mark.xfail(is_big_endian(), reason="Fails on big-endian")
     def test_sanity(self, tmp_path):
 
         # internal version number

Tests/test_file_psd.py

@@ -123,7 +123,7 @@ def test_no_icc_profile():
 
 
 def test_combined_larger_than_size():
-    # The 'combined' sizes of the individual parts is larger than the
+    # The combined size of the individual parts is larger than the
     # declared 'size' of the extra data field, resulting in a backwards seek.
 
     # If we instead take the 'size' of the extra data field as the source of truth,