Merge remote-tracking branch 'origin/main' into simplify

.flake8

@@ -5,4 +5,4 @@ ignore = E203,E501,E741,W503,W604,N817,N814,VNE001,VNE002,VNE003,N802,SIM105,P10
 exclude = build,sample-files,dist,.benchmarks,.git,.github,.mypy_cache,.pytest_cache,.tox
 per-file-ignores =
     tests/*: ASS001,PT011,B011,T001,T201
-    make_changelog.py:T001,T201
+    make_release.py:T001,T201

.github/ISSUE_TEMPLATE/bug-report.md

@@ -17,7 +17,7 @@ Which environment were you using when you encountered the problem?
 $ python -m platform
 # TODO: Your output goes here
 
-$ python -c "import pypdf;print(pypdf.__version__)"
+$ python -c "import pypdf;print(pypdf._debug_versions)"
 # TODO: Your output goes here
 ```
 
@@ -36,4 +36,6 @@ better. Let us know if we may add them to our tests!
 
 This is the complete Traceback I see:
 
-TODO
+```
+# TODO: Your Traceback goes here (if applicable)
+```

.github/ISSUE_TEMPLATE/feature-request.md

@@ -9,7 +9,7 @@ assignees: MartinThoma
 
 ## Explanation
 
-Explain briefly what you want to achive.
+Explain briefly what you want to achieve.
 
 ## Code Example
 

.github/workflows/github-ci.yaml

@@ -14,19 +14,28 @@ on:
     paths-ignore:
       - '**/*.md'
       - '**/*.rst'
+  workflow_dispatch:
 
 jobs:
   tests:
     name: pytest on ${{ matrix.python-version }}
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
-        use-cryptodome: [""]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+        use-crypto-lib: ["pycryptodome"]
         include:
+          - python-version: "3.9"
+            use-crypto-lib: "cryptography"
           - python-version: "3.10"
-            use-cryptodome: "false"
+            use-crypto-lib: ""
     steps:
+    - name: Update APT packages
+      run:
+        sudo apt-get update
+    - name: Install APT dependencies
+      run:
+        sudo apt-get install ghostscript
     - name: Checkout Code
       uses: actions/checkout@v3
       with:
@@ -39,16 +48,17 @@ jobs:
         key: cache-downloaded-files
     - name: Setup Python
       uses: actions/setup-python@v4
-      if: matrix.python-version != '3.11'
+      if: matrix.python-version == '3.6' || matrix.python-version == '3.7' || matrix.python-version == '3.8' || matrix.python-version == '3.9' || matrix.python-version == '3.10'
       with:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
         cache-dependency-path: '**/requirements/ci.txt'
-    - name: Setup Python (3.11)
+    - name: Setup Python (3.11+)
       uses: actions/setup-python@v4
-      if: matrix.python-version == '3.11'
+      if: matrix.python-version == '3.11' || matrix.python-version == '3.12'
       with:
         python-version: ${{ matrix.python-version }}
+        allow-prereleases: true
         cache: 'pip'
         cache-dependency-path: '**/requirements/ci-3.11.txt'
     - name: Upgrade pip
@@ -57,18 +67,28 @@ jobs:
     - name: Install requirements (Python 3)
       run: |
         pip install -r requirements/ci.txt
-      if: matrix.python-version != '3.11'
-    - name: Install requirements (Python 3.11)
+      if: matrix.python-version == '3.6' || matrix.python-version == '3.7' || matrix.python-version == '3.8' || matrix.python-version == '3.9' || matrix.python-version == '3.10'
+    - name: Install requirements (Python 3.11+)
       run: |
         pip install -r requirements/ci-3.11.txt
-      if: matrix.python-version == '3.11'
-    - name: Remove cryptodome
+      if: matrix.python-version == '3.11' || matrix.python-version == '3.12'
+    - name: Remove pycryptodome and cryptography
       run: |
-        pip uninstall pycryptodome -y
-      if: matrix.use-cryptodome == 'false'
+        pip uninstall pycryptodome cryptography -y
+    - name: Install cryptography
+      run: |
+        pip install cryptography
+      if: matrix.use-crypto-lib == 'cryptography'
+    - name: Install pycryptodome
+      run: |
+        pip install pycryptodome
+      if: matrix.use-crypto-lib == 'pycryptodome'
     - name: Install pypdf
       run: |
         pip install .
+    - name: Prepare
+      run: |
+        python -c "from tests import download_test_pdfs; download_test_pdfs()"
     - name: Test with pytest
       run: |
         python -m coverage run --parallel-mode -m pytest tests -vv

.github/workflows/release.yaml

@@ -11,8 +11,6 @@ on:
 
 jobs:
   build_and_publish:
-    # this doesn't make sense if you don't have the PyPI secret
-    if: github.repository == 'py-pdf/pypdf'
     name: Publish a new version of pypdf
     runs-on: ubuntu-latest
 
@@ -48,15 +46,13 @@ jobs:
           echo "date=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV"
           EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
           echo "tag_body<<$EOF" >> "$GITHUB_ENV"
-          git tag -l "${latest_tag}" --format='%(contents:body)' >> "$GITHUB_ENV"
+          git --no-pager tag -l "${latest_tag}" --format='%(contents:body)' >> "$GITHUB_ENV"
           echo "$EOF" >> "$GITHUB_ENV"
       - name: Create GitHub Release 🚀
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        uses: softprops/action-gh-release@v1
         with:
-          tag_name: ${{ github.ref }}
-          release_name: Version ${{ env.latest_tag }}, ${{ env.date }}
+          tag_name: ${{ env.latest_tag }}
+          name: Version ${{ env.latest_tag }}, ${{ env.date }}
           draft: false
           prerelease: false
           body: ${{ env.tag_body }}

.gitignore

@@ -40,3 +40,6 @@ tests/pdf_cache/
 docs/meta/CHANGELOG.md
 docs/meta/CONTRIBUTORS.md
 extracted-images/
+
+RELEASE_COMMIT_MSG.md
+RELEASE_TAG_MSG.md

.pre-commit-config.yaml

@@ -27,23 +27,30 @@ repos:
     -   id: black
         args: [--target-version, py36]
 -   repo: https://github.com/asottile/blacken-docs
-    rev: 1.15.0
+    rev: 1.16.0
     hooks:
     -   id: blacken-docs
         additional_dependencies: [black==22.1.0]
         exclude: "docs/user/robustness.md"
 -   repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.278'
+    rev: v0.0.285
     hooks:
     -   id: ruff
         args: ['--fix']
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.9.0
+    rev: v3.10.1
     hooks:
     -   id: pyupgrade
         args: [--py36-plus]
 -   repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
     -   id: flake8
         args: ["--ignore", "E,W,F"]
+
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v1.5.1'
+    hooks:
+      - id: mypy
+        files: ^pypdf/.*
+        args: [--ignore-missing-imports]

CHANGELOG.md

@@ -1,5 +1,142 @@
 # CHANGELOG
 
+## Version 3.16.0, 2023-09-10
+
+### Security (SEC)
+-  Infinite recursion caused by IndirectObject clone (#2156)
+
+### New Features (ENH)
+-  Ease access to ViewerPreferences (#2144)
+
+### Bug Fixes (BUG)
+-  Catch the case where w[0] is an IndirectObject instead of an int (#2154)
+-  Cope with indirect objects in filters and remove deprecated code (#2177)
+-  Accept tabs in cmaps (#2174) / cope with extra space (#2151)
+-  Merge pages without resources (#2150)
+-  getcontents() shall return None if contents is NullObject (#2161)
+-  Fix conversion from 1 to LA (#2175)
+
+### Robustness (ROB)
+-  Accept XYZ with no arguments (#2178)
+
+[Full Changelog](https://github.com/py-pdf/pypdf/compare/3.15.5...3.16.0)
+
+## Version 3.15.5, 2023-09-03
+
+### Bug Fixes (BUG)
+-  Cope with missing /I in articles (#2134)
+-  Fix image look-up table in EncodedStreamObject (#2128)
+-  remove_images not operating in sub level forms (#2133)
+
+### Robustness (ROB)
+-  Cope with damaged PDF (#2129)
+
+[Full Changelog](https://github.com/py-pdf/pypdf/compare/3.15.4...3.15.5)
+
+## Version 3.15.4, 2023-08-27
+
+### Performance Improvements (PI)
+-  Making pypdf as fast as pdfrw (#2086)
+
+### Maintenance (MAINT)
+-  Relax typing_extensions version (#2104)
+
+[Full Changelog](https://github.com/py-pdf/pypdf/compare/3.15.3...3.15.4)
+
+## Version 3.15.3, 2023-08-26
+
+### Bug Fixes (BUG)
+-  Check version of crypt provider (#2115)
+-  TypeError: can't concat str to bytes (#2114)
+-  Require flit_core >= 3.9 (#2091)
+
+[Full Changelog](https://github.com/py-pdf/pypdf/compare/3.15.2...3.15.3)
+
+## Version 3.15.2, 2023-08-20
+
+### Security (SEC)
+-  Avoid endless recursion of reading damaged PDF file (#2093)
+
+### Performance Improvements (PI)
+-  Re-use content stream (#2101)
+
+### Maintenance (MAINT)
+-  Make ParseError inherit from PyPdfError (#2097)
+
+[Full Changelog](https://github.com/py-pdf/pypdf/compare/3.15.1...3.15.2)
+
+## Version 3.15.1, 2023-08-13
+
+### Performance Improvements (PI)
+-  optimize _decode_png_prediction (#2068)
+
+### Bug Fixes (BUG)
+-  Fix incorrect tm_matrix in call to visitor_text (#2060)
+-  Writing German characters into form fields (#2047)
+-  Prevent stall when accessing image in corrupted pdf (#2081)
+-  append() fails when articles do not have /T (#2080)
+
+### Robustness (ROB)
+-  Cope with xref not followed by separator (#2083)
+
+[Full Changelog](https://github.com/py-pdf/pypdf/compare/3.15.0...3.15.1)
+
+## Version 3.15.0, 2023-08-06
+
+### New Features (ENH)
+-  Add `level` parameter to compress_content_streams (#2044)
+-  Process /uniHHHH for text_extract (#2043)
+
+### Bug Fixes (BUG)
+-  Fix AnnotationBuilder.link (#2066)
+-  JPX image without ColorSpace  (#2062)
+-  Added check for field /Info when cloning reader document (#2055)
+-  Fix indexed/CMYK images (#2039)
+
+### Maintenance (MAINT)
+-  Cryptography as primary dependency (#2053)
+
+[Full Changelog](https://github.com/py-pdf/pypdf/compare/3.14.0...3.15.0)
+
+## Version 3.14.0, 2023-07-29
+
+### New Features (ENH)
+-  Accelerate image list keys generation (#2014)
+-  Use `cryptography` for encryption/decryption as a fallback for PyCryptodome (#2000)
+-  Extract LaTeX characters (#2016)
+-  ASCIIHexDecode.decode now returns bytes instead of str (#1994)
+
+### Bug Fixes (BUG)
+-  Add RunLengthDecode filter (#2012)
+-  Process /Separation ColorSpace (#2007)
+-  Handle single element ColorSpace list (#2026)
+-  Process lookup decoded as TextStringObjects (#2008)
+
+### Robustness (ROB)
+-  Cope with garbage collector during cloning (#1841)
+
+### Maintenance (MAINT)
+-  Cleanup of annotations (#1745)
+
+[Full Changelog](https://github.com/py-pdf/pypdf/compare/3.13.0...3.14.0)
+
+## Version 3.13.0, 2023-07-23
+
+### New Features (ENH)
+-  Add is_open in outlines in PdfReader and PdfWriter (#1960)
+
+### Bug Fixes (BUG)
+-  Search /DA in hierarchy fields (#2002)
+-  Cope with different ISO date length (#1999)
+-  Decode Black only/CMYK deviceN images (#1984)
+-  Process CMYK in deflate images (#1977)
+
+### Developer Experience (DEV)
+-  Add mypy to pre-commit (#2001)
+-  Release automation (#1991, #1985)
+
+[Full Changelog](https://github.com/py-pdf/pypdf/compare/3.12.2...3.13.0)
+
 ## Version 3.12.2, 2023-07-16
 
 ### Bug Fixes (BUG)

CONTRIBUTING.md

@@ -0,0 +1,23 @@
+Please check the [documentation page dedicated to development](https://pypdf.readthedocs.io/en/stable/dev/intro.html).
+
+## Creating issues / tickets
+
+Please go here: https://github.com/py-pdf/pypdf/issues
+
+Typically you should not send e-mails. E-mails might only reach one person and
+it could go into spam or that person might be busy. Please create issues on
+GitHub instead.
+
+Please use the templates provided.
+
+## Creating Pull Requests
+
+We appreciate if people make PRs, but please be aware that pypdf is used by many
+people. That means:
+
+* We rarely make breaking changes and have a [deprecation process](https://pypdf.readthedocs.io/en/latest/dev/deprecations.html)
+* New features, especially adding to the public interface, typically need to be
+  discussed first.
+
+Before you make bigger changes, rather open an issue to make the suggestion.
+Note which interface changes you want to make.

CONTRIBUTORS.md

@@ -13,11 +13,14 @@ history and [GitHubs 'Contributors' feature](https://github.com/py-pdf/pypdf/gra
 
 * [abyesilyurt](https://github.com/abyesilyurt)
 * [ArkieCoder](https://github.com/ArkieCoder)
+* [Clauss, Christian](https://github.com/cclauss)
 * [DL6ER](https://github.com/DL6ER)
 * [ediamondscience](https://github.com/ediamondscience)
 * [Ermeson, Felipe](https://github.com/FelipeErmeson)
+* [Freitag, François](https://github.com/francoisfreitag)
 * [Górny, Michał](https://github.com/mgorny)
 * [Grillo, Miguel](https://github.com/Ineffable22)
+* [Gutteridge, David H.](https://github.com/dhgutteridge)
 * [Hale, Joseph](https://github.com/thehale)
 * [harshhes](https://github.com/harshhes)
 * [JianzhengLuo](https://github.com/JianzhengLuo)
@@ -43,6 +46,7 @@ history and [GitHubs 'Contributors' feature](https://github.com/py-pdf/pypdf/gra
 * [Rogmann, Sascha](https://github.com/srogmann)
 * [robbiebusinessacc](https://github.com/robbiebusinessacc)
 * [Roder, Thomas](https://github.com/MrTomRod)
+* [Röthenbacher, Thomas](https://github.com/troethe)
 * [sietzeberends](https://github.com/sietzeberends)
 * [Stüber, Timo](https://github.com/omit66)
 * [Thoma, Martin](https://github.com/MartinThoma): Maintainer of pypdf since April 2022. I hope to build a great community with many awesome contributors. [LinkedIn](https://www.linkedin.com/in/martin-thoma/) | [StackOverflow](https://stackoverflow.com/users/562769/martin-thoma) | [Blog](https://martin-thoma.com/)

Makefile

@@ -7,8 +7,9 @@ maint:
 	pyenv local 3.7.9
 	pip-compile -U requirements/docs.in
 
-changelog:
-	python make_changelog.py
+release:
+	python make_release.py
+	git commit -eF RELEASE_COMMIT_MSG.md
 
 upload:
 	make clean