From a3886c2088ad57903bfd6ac223eff4620ee0d1ea Mon Sep 17 00:00:00 2001 From: adam-grant-hendry <59346180+adam-grant-hendry@users.noreply.github.com> Date: Tue, 28 Jun 2022 17:10:21 -0700 Subject: [PATCH] Update Formatting Tools (#182) * refactor(trove-classifiers): update to python 3.7-3.9 * refactor(linters): add `.flake8` and `.codespellrc` configuration files Also add `PullRequest` to `ignore_words.txt` * feat(pyproject): add `pyproject.toml` Currently, this is only use for configuring linters and formatters (no build file specifications are set). Configuration options for `black` and `pydocstyle` are added here in this commit. * feat(pre-commit): add pre-commit * fix(mypy): add `mypy` dependency to `environment.yml` This supports conda test/build environments. * feat(pylint): Update `.pylintrc` Make this match settings specififed in `Makefile`. * feat(test): align checks All checks are placed in appropriate config files. Test runner scripts point to these so that there exists a single source of truth for all configurations and that the tests performed across systems is the same. This may be adjusted, of course, if different settings must be used on a per-system basis. * feat(requirements): add `toml` This package supports reading `pyproject.toml`, which is required for some of our tests. * fix(pre-commit): fix file passing errors `pre-commit` is known to override behavior from config files in certain instances. This commit fixes known issues for: - `isort` - `black` - `mypy` - `pydocstyle` For relevant details, see: isort: + https://jugmac00.github.io/blog/isort-and-pre-commit-a-friendship-with-obstacles/ + https://github.com/PyCQA/isort/issues/885 black: + https://github.com/psf/black/issues/1584 mypy/pydocstyle: + https://github.com/python/mypy/issues/4008#issuecomment-708060733 + https://pre-commit.com/#hooks-pass_filenames * feat(ignores): ignore specific linting errors Ignore linting errors in source code in this PR. The purpose of this PR is to update linting tools. A future PR will correct the errors. This is done to separate concerns. Co-authored-by: Hendry, Adam --- .ci/azure-pipelines.yml | 2 +- .codespellrc | 4 ++ .flake8 | 42 ++++++++++++++++ .isort.cfg | 10 ++++ .pre-commit-config.yaml | 108 ++++++++++++++++++++++++++++++++++++++++ .pycodestyle | 8 +++ .pylintrc | 93 ++++++---------------------------- Makefile | 41 ++++++++------- environment.yml | 2 + ignore_words.txt | 1 + mypy.ini | 1 + pyproject.toml | 29 +++++++++++ requirements_docs.txt | 1 + requirements_test.txt | 7 +++ setup.py | 4 +- 15 files changed, 254 insertions(+), 99 deletions(-) create mode 100644 .codespellrc create mode 100644 .flake8 create mode 100644 .pre-commit-config.yaml create mode 100644 .pycodestyle create mode 100644 pyproject.toml diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml index 7538e638..7949eb49 100644 --- a/.ci/azure-pipelines.yml +++ b/.ci/azure-pipelines.yml @@ -52,7 +52,7 @@ stages: inputs: versionSpec: '3.7' - script: | - pip install codespell pydocstyle + pip install codespell pydocstyle[toml] make doctest displayName: 'Run doctest' diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 00000000..5fcc7728 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,4 @@ +[codespell] +skip = ./.ci/*,./.git/*,./.hypothesis/*,./dist/*,./doc/examples/*,./doc/images/*,./docs/_build/*,./docs/images/*,./tests/tinypages/_build/*,.hypothesis*,*.doctree,*.eot,*.gif,*.html,*.inv,*.ipynb,*.jpg,*.js,*.json,*.mp4,*.mypy_cache/*,*.pickle,*.ply,*.png,*.pyc,*.ttf,*.txt,*.vti,*.vtk,*.vtu,*.woff,*.woff2,*.yml,*/_autosummary/*,*~,*cover,doc/_build/*,flycheck* +ignore-words-list = lod,byteorder,flem,parm,doubleclick,revered,PullRequest +quiet-level = 3 diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..ce7b4297 --- /dev/null +++ b/.flake8 @@ -0,0 +1,42 @@ +[flake8] +max-line-length = 88 +exclude = + __pycache__, + .venv, + .cache, + .eggs + .git, + .tox, + *.egg-info, + *.pyc, + *.pyi, + build, + dist, + # This is adopted from VTK + pyvistaqt/rwi.py, + # Only lint source files + tests/*.py, + docs/*.py, + # Ignore errors here in this commit (fix in future PR) + pyvistaqt/*.py, + setup.py +max-complexity = 10 +doctests = true +extend-ignore = + # whitespace before ':' + E203, + # line break before binary operator + W503, + # line length too long + E501, + # do not assign a lambda expression, use a def + E731, + # missing class docstring; use ``__init__`` docstring instead + D101, + # missing docstring in magic method + D105, + # Qt uses camelCase + N802 +per-file-ignores = + # Allow re-export of modules at package level + __init__.py:F401 \ No newline at end of file diff --git a/.isort.cfg b/.isort.cfg index ba2778dc..1567719e 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,16 @@ [settings] +profile=black multi_line_output=3 include_trailing_comma=True force_grid_wrap=0 use_parentheses=True line_length=88 +ensure_newline_before_comments=True +extend_skip_glob=tests/*.py,docs/conf.py,setup.py,pyvistaqt/rwi.py +# `pre-comment` doesn't see skips; `filter_files=True` forces it +# to see these files +# +# See: +# - https://jugmac00.github.io/blog/isort-and-pre-commit-a-friendship-with-obstacles/ +# - https://github.com/PyCQA/isort/issues/885 +filter_files=True \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..38d3fc70 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,108 @@ +repos: +- repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + args: [ + "--config=pyproject.toml" + ] + +- repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + args: [ + "--check", + "--settings=.isort.cfg" + ] + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.961 + hooks: + - id: mypy + # `pass_filenames` is used to overcome the "duplicate module" + # error from occuring. We are explicitly passing a 'txt' + # file to search. This setting tells `pre-commit` to not do a + # search and pass files to check to mypy, like it normally does. + # + # See: + # - https://github.com/python/mypy/issues/4008#issuecomment-708060733 + # - https://pre-commit.com/#hooks-pass_filenames + language: system + pass_filenames: false + args: [ + "--config-file", + "mypy.ini", + "@mypy_checklist.txt" + ] + +- repo: https://gitlab.com/PyCQA/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + additional_dependencies: [ + "flake8-black==0.3.2", + "flake8-isort==4.1.1", + "flake8-quotes==3.3.1", + ] + args: [ + "--config=.flake8" + ] + +- repo: https://github.com/codespell-project/codespell + rev: v2.1.0 + hooks: + - id: codespell + args: [ + "docs examples examples_flask pyvista tests", + "*.py *.rst *.md", + ] + +- repo: https://github.com/pycqa/pydocstyle + rev: 6.1.1 + hooks: + - id: pydocstyle + additional_dependencies: [toml==0.10.2] + # We use the 'match' and do not want pre-commit to pass + # globbed files + pass_filenames: false + args: [ + "--config=pyproject.toml", + ] + +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: check-merge-conflict + - id: debug-statements + - id: no-commit-to-branch + args: [--branch, main] + +# this validates our github workflow files +- repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.16.1 + hooks: + - id: check-github-workflows + +- repo: local + hooks: + - id: pylint + name: pylint + entry: pylint + language: system + types: [python] + args: + [ + "-rn", # Only display messages + "-sn", # Don't display the score + "--rcfile=.pylintrc", # Specify rc file + ] + - id: pycodestyle + name: pycodestyle + entry: pycodestyle + language: system + types: [python] + args: + [ + "--config=./\\.pycodestyle", + ] \ No newline at end of file diff --git a/.pycodestyle b/.pycodestyle new file mode 100644 index 00000000..a2b33ce6 --- /dev/null +++ b/.pycodestyle @@ -0,0 +1,8 @@ +[pycodestyle] +ignore = E501, + E203, + W503 +exclude = tests/*.py, + docs/*.py, + rwi.py +filename = pyvistaqt/*.py \ No newline at end of file diff --git a/.pylintrc b/.pylintrc index a0629152..5de8a04a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,18 +3,25 @@ # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. -extension-pkg-whitelist=vtk +extension-pkg-whitelist=vtk, + PyQt5, + PySide2, + PyQt6, + PySide6 # Specify a score threshold to be exceeded before program exits with error. fail-under=10 # Add files or directories to the blacklist. They should be base names, not # paths. -ignore=CVS +ignore=rwi.py, + conf.py, + conftest.py, + setup.py # ignore and fix in future PR # Add files or directories matching the regex patterns to the blacklist. The # regex matches against base names, not paths. -ignore-patterns= +ignore-patterns=(?=(coverage|test_|rwi)).*[.]py # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). @@ -60,17 +67,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, +disable=raw-checker-failed, bad-inline-option, locally-disabled, file-ignored, @@ -78,71 +75,13 @@ disable=print-statement, useless-suppression, deprecated-pragma, use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, - bad-continuation, arguments-differ, no-name-in-module, - no-member + no-member, + # Redundant alias imports required for type hinting by PEP 484 + useless-import-alias, + # Qt uses PascalCase + invalid-name, # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/Makefile b/Makefile index 4fa729b3..ceaf59ed 100644 --- a/Makefile +++ b/Makefile @@ -7,16 +7,19 @@ PYLINT_DIRS ?= ./pyvistaqt/ MYPY_DIRS ?= "mypy_checklist.txt" FLAKE8_DIRS ?= ./pyvistaqt/ CODESPELL_DIRS ?= ./ -CODESPELL_SKIP ?= "*.json,*.pyc,*.txt,*.gif,*.png,*.jpg,*.ply,*.vtk,*.vti,*.js,*.html,*.doctree,*.ttf,*.woff,*.woff2,*.eot,*.mp4,*.inv,*.pickle,*.ipynb,flycheck*,./.git/*,./.hypothesis/*,*.yml,./docs/_build/*,./docs/images/*,./dist/*,./.ci/*" -CODESPELL_IGNORE ?= "ignore_words.txt" - -EXTRA_BLACK_OPTIONS ?= --exclude rwi.py -EXTRA_ISORT_OPTIONS ?= --skip=rwi.py -EXTRA_PYLINT_OPTIONS ?= --ignore=rwi.py -EXTRA_PYCODESTYLE_OPTIONS ?= --ignore="E501,E203,W503" --exclude=rwi.py -EXTRA_MYPY_OPTIONS ?= --follow-imports=skip -EXTRA_FLAKE8_OPTIONS ?= --ignore="E501,E203,W503" --exclude=rwi.py -EXTRA_PYDOCSTYLE_OPTIONS = --match='(?!(test_|rwi)).*\.py' +PYDOCSTYLE_DIRS ?= ./pyvistaqt/ +COVERAGE_DIRS ?= ./pyvistaqt/ +COVERAGE_HTML_DIRS ?= ./pyvistaqt/ +COVERAGE_XML_DIRS ?= ./pyvistaqt/ + +EXTRA_CODESPELL_OPTIONS ?= --config .codespellrc +EXTRA_BLACK_OPTIONS ?= --config pyproject.toml +EXTRA_ISORT_OPTIONS ?= --check --settings=.isort.cfg +EXTRA_PYLINT_OPTIONS ?= -rn -sn --rcfile=.pylintrc +EXTRA_PYCODESTYLE_OPTIONS ?= --config=.pycodestyle +EXTRA_MYPY_OPTIONS ?= --config-file mypy.ini +EXTRA_FLAKE8_OPTIONS ?= --config=.flake8 +EXTRA_PYDOCSTYLE_OPTIONS = --config=pyproject.toml all: srcstyle doctest @@ -26,15 +29,15 @@ doctest: codespell pydocstyle black: @echo "Running black" - @black --check $(BLACK_DIRS) $(EXTRA_BLACK_OPTIONS) + @black $(BLACK_DIRS) $(EXTRA_BLACK_OPTIONS) isort: @echo "Running isort" - @isort --check $(ISORT_DIRS) $(EXTRA_ISORT_OPTIONS) + @isort $(ISORT_DIRS) $(EXTRA_ISORT_OPTIONS) pylint: @echo "Running pylint" - @pylint $(PYLINT_DIRS) --rcfile=.pylintrc $(EXTRA_PYLINT_OPTIONS) + @pylint $(PYLINT_DIRS) $(EXTRA_PYLINT_OPTIONS) pycodestyle: @echo "Running pycodestyle" @@ -42,7 +45,7 @@ pycodestyle: mypy: @echo "Running mypy" - @mypy --config-file mypy.ini @$(MYPY_DIRS) $(EXTRA_MYPY_OPTIONS) + @mypy @$(MYPY_DIRS) $(EXTRA_MYPY_OPTIONS) flake8: @echo "Running flake8" @@ -50,20 +53,20 @@ flake8: codespell: @echo "Running codespell" - @codespell $(CODESPELL_DIRS) -S $(CODESPELL_SKIP) -I $(CODESPELL_IGNORE) + @codespell $(CODESPELL_DIRS) $(EXTRA_CODESPELL_OPTIONS) pydocstyle: @echo "Running pydocstyle" - @pydocstyle pyvistaqt $(EXTRA_PYDOCSTYLE_OPTIONS) + @pydocstyle $(PYDOCSTYLE_DIRS) $(EXTRA_PYDOCSTYLE_OPTIONS) coverage: @echo "Running coverage" - @pytest -v --cov pyvistaqt + @pytest -v --cov $(COVERAGE_DIRS) coverage-xml: @echo "Reporting XML coverage" - @pytest -v --cov pyvistaqt --cov-report xml + @pytest -v --cov $(COVERAGE_XML_DIRS) --cov-report xml coverage-html: @echo "Reporting HTML coverage" - @pytest -v --cov pyvistaqt --cov-report html + @pytest -v --cov $(COVERAGE_HTML_DIRS) --cov-report html diff --git a/environment.yml b/environment.yml index c0999b85..6dfe456f 100644 --- a/environment.yml +++ b/environment.yml @@ -5,12 +5,14 @@ dependencies: - python<3.9 - codecov - ipython + - mypy - numpy - pytest - pytest-cov - pytest-qt - qtpy>=1.9.0 - scooby>=0.5.1 + - toml - pip - pip: - vtk diff --git a/ignore_words.txt b/ignore_words.txt index 08703815..35430c91 100644 --- a/ignore_words.txt +++ b/ignore_words.txt @@ -1,3 +1,4 @@ lod byteorder flem +PullRequest \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index 5fbf65b4..2b99d960 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,5 @@ [mypy] +ignore_errors = True disallow_untyped_defs = True [mypy-vtk.*] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..32e2a562 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[tool.black] +line-length = 88 +skip-string-normalization = true +target-version = ["py39"] +# `pre-comment` doesn't see skips; `force-exclude` forces it +# to see these files +# +# See: +# - https://github.com/psf/black/issues/1584 +force-exclude = ''' +( + docs/conf\.py + | pyvistaqt/rwi\.py + | tests/.*\.py + | setup.py +) +''' +check = true + +[tool.pydocstyle] +match = ''' +(?! + ( + | tests/ + | docs/ + | rwi + ).*\.py +) +''' \ No newline at end of file diff --git a/requirements_docs.txt b/requirements_docs.txt index a58307e0..71932dd1 100644 --- a/requirements_docs.txt +++ b/requirements_docs.txt @@ -14,3 +14,4 @@ sphinx-notfound-page==0.8 sphinx-rtd-theme==1.0.0 sphinxcontrib-napoleon==0.7 sphinxcontrib-websupport==1.2.4 +toml==0.10.2 \ No newline at end of file diff --git a/requirements_test.txt b/requirements_test.txt index 323e5d68..c5537c53 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -16,3 +16,10 @@ pytest-qt==4.1.0 pyvista==0.34.1 QtPy==2.1.0 scooby==0.5.12 +flake8-black==0.3.2 +flake8-isort==4.1.1 +flake8-quotes==3.3.1 +check-jsonschema==0.16.1 +pre-commit==2.19.0 +pre-commit-hooks==4.3.0 +toml==0.10.2 \ No newline at end of file diff --git a/setup.py b/setup.py index 23a0a23d..240fc290 100644 --- a/setup.py +++ b/setup.py @@ -33,14 +33,14 @@ 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX', 'Operating System :: MacOS', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], url='https://github.com/pyvista/pyvistaqt', keywords='vtk numpy plotting mesh qt', - python_requires='>=3.6.*', + python_requires='>=3.7.*', install_requires=[ 'pyvista>=0.32.0', 'QtPy>=1.9.0',