From 8ba58672b700c233d86bd7d9449443e6bfed6669 Mon Sep 17 00:00:00 2001 From: Bruce Dawson Date: Fri, 24 Jun 2022 13:41:18 -0700 Subject: [PATCH 1/8] Avoid Win32 multiprocessing limits Creating a multiprocessing Pool with too many processes can hit ValueError exceptions or hangs or both. The number that counts as "too many" depends on the Python version so this change uses 56 as a guaranteed safe limit. Details in issue 6965 Note that this only avoids the issue if an explicit jobs count is not passed in. --- pylint/lint/run.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pylint/lint/run.py b/pylint/lint/run.py index 27a4d9d84d..3bc6f7c192 100644 --- a/pylint/lint/run.py +++ b/pylint/lint/run.py @@ -190,6 +190,11 @@ def __init__( linter.set_option("jobs", 1) elif linter.config.jobs == 0: linter.config.jobs = _cpu_count() + if sys.platform == 'win32': + # Using too many child processes in Python 3 hits either hangs or a + # ValueError exception, and, has diminishing returns. Clamp to 56 to + # give margin for error. + jobs = min(jobs, 56) if self._output: try: From a8b2ad8d2b61b603f321d8b559f1a1b2b53b6f2a Mon Sep 17 00:00:00 2001 From: Bruce Dawson Date: Fri, 24 Jun 2022 13:48:06 -0700 Subject: [PATCH 2/8] Add note to index.rst --- doc/whatsnew/2/2.15/index.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/whatsnew/2/2.15/index.rst b/doc/whatsnew/2/2.15/index.rst index e8ce187909..5da8d10606 100644 --- a/doc/whatsnew/2/2.15/index.rst +++ b/doc/whatsnew/2/2.15/index.rst @@ -94,3 +94,6 @@ Internal changes let the functional test fail with a default value. Refs #6891 + +* Fixed an issue where many-core Windows machines (>~60 logical processors) would hang when + using the default jobs count. From 75b2d6c5ec1637cadd3f90b37c7af5668a176e3c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 Jun 2022 20:50:54 +0000 Subject: [PATCH 3/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pylint/lint/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pylint/lint/run.py b/pylint/lint/run.py index 3bc6f7c192..8d4559221e 100644 --- a/pylint/lint/run.py +++ b/pylint/lint/run.py @@ -190,11 +190,11 @@ def __init__( linter.set_option("jobs", 1) elif linter.config.jobs == 0: linter.config.jobs = _cpu_count() - if sys.platform == 'win32': + if sys.platform == "win32": # Using too many child processes in Python 3 hits either hangs or a # ValueError exception, and, has diminishing returns. Clamp to 56 to # give margin for error. - jobs = min(jobs, 56) + min(jobs, 56) if self._output: try: From 252fc65163a4c3aff2555c40e00ac4a917f166b4 Mon Sep 17 00:00:00 2001 From: Bruce Dawson Date: Fri, 24 Jun 2022 15:09:05 -0700 Subject: [PATCH 4/8] Merging with head --- pylint/lint/base_options.py | 3 ++- pylint/lint/run.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pylint/lint/base_options.py b/pylint/lint/base_options.py index 25327e635e..d909e39c59 100644 --- a/pylint/lint/base_options.py +++ b/pylint/lint/base_options.py @@ -244,7 +244,8 @@ def _make_linter_options(linter: PyLinter) -> Options: "short": "j", "default": 1, "help": "Use multiple processes to speed up Pylint. Specifying 0 will " - "auto-detect the number of processors available to use.", + "auto-detect the number of processors available to use, and will cap " + "the count on Windows to avoid hangs.", }, ), ( diff --git a/pylint/lint/run.py b/pylint/lint/run.py index 8d4559221e..28c3082477 100644 --- a/pylint/lint/run.py +++ b/pylint/lint/run.py @@ -82,6 +82,11 @@ def _cpu_count() -> int: cpu_count = multiprocessing.cpu_count() else: cpu_count = 1 + if sys.platform == 'win32': + # Using too many child processes in Python 3 hits either hangs or a + # ValueError exception, and, has diminishing returns. Clamp to 56 to + # give margin for error. + cpu_count = min(cpu_count, 56) if cpu_share is not None: return min(cpu_share, cpu_count) return cpu_count @@ -190,11 +195,6 @@ def __init__( linter.set_option("jobs", 1) elif linter.config.jobs == 0: linter.config.jobs = _cpu_count() - if sys.platform == "win32": - # Using too many child processes in Python 3 hits either hangs or a - # ValueError exception, and, has diminishing returns. Clamp to 56 to - # give margin for error. - min(jobs, 56) if self._output: try: From a47759db1452cd9358bf06ca840749b1c0d03339 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 Jun 2022 22:11:24 +0000 Subject: [PATCH 5/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pylint/lint/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylint/lint/run.py b/pylint/lint/run.py index 28c3082477..8759b7e471 100644 --- a/pylint/lint/run.py +++ b/pylint/lint/run.py @@ -82,7 +82,7 @@ def _cpu_count() -> int: cpu_count = multiprocessing.cpu_count() else: cpu_count = 1 - if sys.platform == 'win32': + if sys.platform == "win32": # Using too many child processes in Python 3 hits either hangs or a # ValueError exception, and, has diminishing returns. Clamp to 56 to # give margin for error. From 99f24c069ad369334b4d417ec4e1d5407c1adc3a Mon Sep 17 00:00:00 2001 From: Bruce Dawson Date: Fri, 24 Jun 2022 15:25:13 -0700 Subject: [PATCH 6/8] Doc changes and comment change --- doc/whatsnew/2/2.14/index.rst | 9 +++++++++ doc/whatsnew/2/2.15/index.rst | 2 ++ pylint/lint/run.py | 4 +--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/doc/whatsnew/2/2.14/index.rst b/doc/whatsnew/2/2.14/index.rst index e21437fcdf..087d74af3d 100644 --- a/doc/whatsnew/2/2.14/index.rst +++ b/doc/whatsnew/2/2.14/index.rst @@ -10,3 +10,12 @@ summary.rst full.rst + + +Internal changes +================ + +* Fixed an issue where many-core Windows machines (>~60 logical processors) would hang when + using the default jobs count. + + Closes #6965 diff --git a/doc/whatsnew/2/2.15/index.rst b/doc/whatsnew/2/2.15/index.rst index 5da8d10606..a62ddebbfe 100644 --- a/doc/whatsnew/2/2.15/index.rst +++ b/doc/whatsnew/2/2.15/index.rst @@ -97,3 +97,5 @@ Internal changes * Fixed an issue where many-core Windows machines (>~60 logical processors) would hang when using the default jobs count. + + Closes #6965 diff --git a/pylint/lint/run.py b/pylint/lint/run.py index 8759b7e471..52f40d3f71 100644 --- a/pylint/lint/run.py +++ b/pylint/lint/run.py @@ -83,9 +83,7 @@ def _cpu_count() -> int: else: cpu_count = 1 if sys.platform == "win32": - # Using too many child processes in Python 3 hits either hangs or a - # ValueError exception, and, has diminishing returns. Clamp to 56 to - # give margin for error. + # See also https://github.com/python/cpython/issues/94242 cpu_count = min(cpu_count, 56) if cpu_share is not None: return min(cpu_share, cpu_count) From 3d19c0dd2a0b2540e63a236eea41a0d909dabc18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Sat, 25 Jun 2022 09:34:59 +0200 Subject: [PATCH 7/8] Move changelog --- doc/whatsnew/2/2.14/full.rst | 5 +++++ doc/whatsnew/2/2.14/index.rst | 9 --------- doc/whatsnew/2/2.15/index.rst | 5 ----- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/doc/whatsnew/2/2.14/full.rst b/doc/whatsnew/2/2.14/full.rst index a0d5e7bc8a..cd0a5aa158 100644 --- a/doc/whatsnew/2/2.14/full.rst +++ b/doc/whatsnew/2/2.14/full.rst @@ -9,6 +9,11 @@ Release date: TBA Closes #7006 +* Fixed an issue where many-core Windows machines (>~60 logical processors) would hang when + using the default jobs count. + + Closes #6965 + What's New in Pylint 2.14.3? ---------------------------- Release date: 2022-06-18 diff --git a/doc/whatsnew/2/2.14/index.rst b/doc/whatsnew/2/2.14/index.rst index 087d74af3d..e21437fcdf 100644 --- a/doc/whatsnew/2/2.14/index.rst +++ b/doc/whatsnew/2/2.14/index.rst @@ -10,12 +10,3 @@ summary.rst full.rst - - -Internal changes -================ - -* Fixed an issue where many-core Windows machines (>~60 logical processors) would hang when - using the default jobs count. - - Closes #6965 diff --git a/doc/whatsnew/2/2.15/index.rst b/doc/whatsnew/2/2.15/index.rst index a62ddebbfe..e8ce187909 100644 --- a/doc/whatsnew/2/2.15/index.rst +++ b/doc/whatsnew/2/2.15/index.rst @@ -94,8 +94,3 @@ Internal changes let the functional test fail with a default value. Refs #6891 - -* Fixed an issue where many-core Windows machines (>~60 logical processors) would hang when - using the default jobs count. - - Closes #6965 From 7c709da11e8f9103310cd23d9ee5c029d5cc2fbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Sat, 25 Jun 2022 09:37:01 +0200 Subject: [PATCH 8/8] Add coverage pragma --- pylint/lint/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylint/lint/run.py b/pylint/lint/run.py index 52f40d3f71..b8923aac5f 100644 --- a/pylint/lint/run.py +++ b/pylint/lint/run.py @@ -84,7 +84,7 @@ def _cpu_count() -> int: cpu_count = 1 if sys.platform == "win32": # See also https://github.com/python/cpython/issues/94242 - cpu_count = min(cpu_count, 56) + cpu_count = min(cpu_count, 56) # pragma: no cover if cpu_share is not None: return min(cpu_share, cpu_count) return cpu_count